List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:in.dream_lab.goffish.hama.PartitionsLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.debug("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { //NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); //LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); int partitionID = Integer.parseInt(vertexValue[1]); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/* w ww . j av a 2 s .co m*/ partitionVertices.add(stringInput); } else { LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 2; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); _edges.add(e); } vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } // End of first superstep. peer.sync(); LOG.debug("Second Superstep in Reader " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { ControlMessage ctrlMessage = (ControlMessage) msg.getControlInfo(); String msgString = ctrlMessage.getVertexValues(); // String msgStringArr[] = msgString.split(","); // for (int i = 0; i < msgStringArr.length; i++) { String vertexInfo[] = msgString.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexInfo[0])); Vertex<V, E, LongWritable, LongWritable> source = (Vertex<V, E, LongWritable, LongWritable>) vertexMap .get(vertexID); if (source == null) { source = new Vertex<V, E, LongWritable, LongWritable>(vertexID); vertexMap.put(source.getVertexId(), source); } for (int j = 2; j < vertexInfo.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexInfo[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); source.addEdge(e); _edges.add(e); } } //} LOG.debug("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexId(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.debug("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.debug("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); assert (sink != null); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { LOG.info("Creating vertices"); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); }//from ww w .j a v a2 s . c om LOG.info("Sending Vertices to respective partitions"); LOG.info("Received all vertices"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
@SuppressWarnings("unchecked") Vertex<V, E, LongWritable, LongWritable> createVertex(String JSONString) { JSONArray JSONInput = (JSONArray) JSONValue.parse(JSONString); LongWritable sourceID = new LongWritable(Long.valueOf(JSONInput.get(0).toString())); assert (vertexMap.get(sourceID) == null); Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>(sourceID); //fix this//w w w. j a v a 2s . c o m //assumed value of jsonMap= "key1:type1:value1$ key2:type2:value2$....." //type could be Long or String or Double String jsonMap = JSONInput.get(1).toString(); String[] vprop = jsonMap.split(Pattern.quote("$")); //key,value property pairs for a vertex MapWritable vertexMap = new MapWritable(); for (int i = 0; i < vprop.length; i++) { String[] map = vprop[i].split(Pattern.quote(":")); Text key = new Text(map[0]); //FIXME:assuming String values for now Text value = new Text(map[2]); vertexMap.put(key, value); } V vertexValue = (V) vertexMap; vertex.setValue(vertexValue); JSONArray edgeList = (JSONArray) JSONInput.get(2); for (Object edgeInfo : edgeList) { Object edgeValues[] = ((JSONArray) edgeInfo).toArray(); LongWritable sinkID = new LongWritable(Long.valueOf(edgeValues[0].toString())); LongWritable edgeID = new LongWritable(Long.valueOf(edgeValues[1].toString())); //fix this //same format as vertex String[] eprop = edgeValues[2].toString().split(Pattern.quote("$")); MapWritable edgeMap = new MapWritable(); for (int i = 0; i < eprop.length; i++) { String[] map = eprop[i].split(Pattern.quote(":")); Text key = new Text(map[0]); //FIXME:assuming String values for now Text value = new Text(map[2]); edgeMap.put(key, value); } Edge<E, LongWritable, LongWritable> edge = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); E edgeValue = (E) edgeMap; edge.setValue(edgeValue); vertex.addEdge(edge); } return vertex; }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());/*from w ww . ja v a 2s .c o m*/ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.outEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexID()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionID()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexID(), subgraph.getSubgraphID()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.LongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { //NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); //LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); int partitionID = Integer.parseInt(vertexValue[1]); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/*from w ww .j av a 2 s . c o m*/ partitionVertices.add(stringInput); } else { LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 2; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); _edges.add(e); } vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); System.out.println("To " + partitionID + " " + vertices.size()); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } // End of first superstep. System.gc(); peer.sync(); LOG.info("Second Superstep in Reader " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { ControlMessage ctrlMessage = (ControlMessage) msg.getControlInfo(); String msgString = ctrlMessage.getVertexValues(); // String msgStringArr[] = msgString.split(","); // for (int i = 0; i < msgStringArr.length; i++) { String vertexInfo[] = msgString.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexInfo[0])); Vertex<V, E, LongWritable, LongWritable> source = (Vertex<V, E, LongWritable, LongWritable>) vertexMap .get(vertexID); if (source == null) { source = new Vertex<V, E, LongWritable, LongWritable>(vertexID); vertexMap.put(source.getVertexID(), source); } for (int j = 2; j < vertexInfo.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexInfo[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); source.addEdge(e); _edges.add(e); } } //} LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.info("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Set<LongWritable> visited = new HashSet<LongWritable>(); Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());/* w ww .j a v a 2 s. co m*/ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.outEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexID()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionID()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexID(), subgraph.getSubgraphID()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.LongTextJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { // Map of partitionID,vertex that do not belong to this partition Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); JSONArray JSONInput = (JSONArray) JSONValue.parse(StringJSONInput); int partitionID = Integer.parseInt(JSONInput.get(1).toString()); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }// ww w .j a va 2 s. co m partitionVertices.add(StringJSONInput); } else { Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } //End of first SuperStep peer.sync(); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { String JSONVertex = msg.getControlInfo().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(JSONVertex); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }