List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:in.dream_lab.goffish.hama.FullInfoNonSplitReader.java
License:Apache License
private void createVertex(String stringInput) { String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[1])); int partitionID = Integer.parseInt(vertexValue[0]); LongWritable vertexSubgraphID = new LongWritable(Long.parseLong(vertexValue[2])); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = (Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>) partition .getSubgraph(vertexSubgraphID); if (subgraph == null) { subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>(partitionID, vertexSubgraphID);/*from w w w.j a v a2 s. c o m*/ partition.addSubgraph(subgraph); } List<IEdge<E, LongWritable, LongWritable>> _adjList = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); for (int j = 3; j < vertexValue.length; j++) { if (j + 3 > vertexValue.length) { LOG.debug("Incorrect length of line for vertex " + vertexID); } LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable sinkSubgraphID = new LongWritable(Long.parseLong(vertexValue[j + 1])); int sinkPartitionID = Integer.parseInt(vertexValue[j + 2]); j += 2; LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); _adjList.add(e); if (sinkPartitionID != peer.getPeerIndex() && subgraph.getVertexById(sinkID) == null) { // this is a remote vertex IRemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = new RemoteVertex<>(sinkID, sinkSubgraphID); // Add it to the same subgraph, as this is part of weakly connected // component subgraph.addVertex(sink); } } subgraph.addVertex(createVertexInstance(vertexID, _adjList)); }
From source file:in.dream_lab.goffish.hama.FullInfoSplitReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String stringInput = pair.getValue().toString(); // pid is the first column and its range is 0 to max pid int partitionID = Integer.parseInt(stringInput.substring(0, stringInput.indexOf('\t'))); LOG.debug("partitionID = " + partitionID); if (partitionID != peer.getPeerIndex()) { // send vertex to its correct partition Message<K, M> msg = new Message<>(); msg.setMessageType(Message.MessageType.VERTEX); ControlMessage ctrl = new ControlMessage(); ctrl.setTransmissionType(IControlMessage.TransmissionType.VERTEX); ctrl.addextraInfo(stringInput.getBytes()); msg.setControlInfo(ctrl);//w ww. j a v a 2 s . co m peer.send(peer.getPeerName(partitionID), msg); } else { // belongs to this partition createVertex(stringInput); } } peer.sync(); Message<K, M> msg; //recieve all incoming vertices while ((msg = peer.getCurrentMessage()) != null) { ControlMessage receivedCtrl = (ControlMessage) msg.getControlInfo(); createVertex(new String(receivedCtrl.getExtraInfo().iterator().next().copyBytes())); } // broadcast all subgraphs belonging to this partition Message<K, M> subgraphMapppingMessage = new Message<>(); subgraphMapppingMessage.setMessageType(Message.MessageType.CUSTOM_MESSAGE); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); controlInfo.setPartitionID(peer.getPeerIndex()); subgraphMapppingMessage.setControlInfo(controlInfo); for (ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph : partition.getSubgraphs()) { byte subgraphIDbytes[] = Longs.toByteArray(subgraph.getSubgraphId().get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphMapppingMessage); peer.sync(); Message<K, M> subgraphMappingInfoMessage; while ((subgraphMappingInfoMessage = peer.getCurrentMessage()) != null) { ControlMessage receivedCtrl = (ControlMessage) subgraphMappingInfoMessage.getControlInfo(); Integer partitionID = receivedCtrl.getPartitionID(); for (BytesWritable rawSubgraphID : receivedCtrl.getExtraInfo()) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(rawSubgraphID.copyBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.FullInfoSplitReader.java
License:Apache License
private void createVertex(String stringInput) { // belongs to this partition String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[1])); int partitionID = Integer.parseInt(vertexValue[0]); LongWritable vertexSubgraphID = new LongWritable(Long.parseLong(vertexValue[2])); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = (Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>) partition .getSubgraph(vertexSubgraphID); if (subgraph == null) { subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>(partitionID, vertexSubgraphID);//from w ww . ja v a2 s . c o m partition.addSubgraph(subgraph); } List<IEdge<E, LongWritable, LongWritable>> _adjList = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); for (int j = 3; j < vertexValue.length; j++) { if (j + 3 > vertexValue.length) { LOG.debug("Incorrect length of line for vertex " + vertexID); } LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable sinkSubgraphID = new LongWritable(Long.parseLong(vertexValue[j + 1])); int sinkPartitionID = Integer.parseInt(vertexValue[j + 2]); j += 2; LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); _adjList.add(e); if (sinkPartitionID != peer.getPeerIndex() && subgraph.getVertexById(sinkID) == null) { // this is a remote vertex IRemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = new RemoteVertex<>(sinkID, sinkSubgraphID); // Add it to the same subgraph, as this is part of weakly connected // component subgraph.addVertex(sink); } } subgraph.addVertex(createVertexInstance(vertexID, _adjList)); }
From source file:in.dream_lab.goffish.hama.FullInfoSplitReaderInt.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, IntWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String stringInput = pair.getValue().toString(); // pid is the first column and its range is 0 to max pid int partitionID = Integer.parseInt(stringInput.substring(0, stringInput.indexOf('\t'))); LOG.debug("partitionID = " + partitionID); if (partitionID != peer.getPeerIndex()) { // send vertex to its correct partition Message<K, M> msg = new Message<>(); msg.setMessageType(Message.MessageType.VERTEX); ControlMessage ctrl = new ControlMessage(); ctrl.setTransmissionType(IControlMessage.TransmissionType.VERTEX); ctrl.addextraInfo(stringInput.getBytes()); msg.setControlInfo(ctrl);/* ww w . j a va 2 s . c o m*/ peer.send(peer.getPeerName(partitionID), msg); } else { // belongs to this partition createVertex(stringInput); } } peer.sync(); Message<K, M> msg; //recieve all incoming vertices while ((msg = peer.getCurrentMessage()) != null) { ControlMessage receivedCtrl = (ControlMessage) msg.getControlInfo(); createVertex(new String(receivedCtrl.getExtraInfo().iterator().next().copyBytes())); } // broadcast all subgraphs belonging to this partition Message<K, M> subgraphMapppingMessage = new Message<>(); subgraphMapppingMessage.setMessageType(Message.MessageType.CUSTOM_MESSAGE); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); controlInfo.setPartitionID(peer.getPeerIndex()); subgraphMapppingMessage.setControlInfo(controlInfo); for (ISubgraph<S, V, E, LongWritable, IntWritable, LongWritable> subgraph : partition.getSubgraphs()) { byte subgraphIDbytes[] = Longs.toByteArray(subgraph.getSubgraphId().get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphMapppingMessage); peer.sync(); Message<K, M> subgraphMappingInfoMessage; while ((subgraphMappingInfoMessage = peer.getCurrentMessage()) != null) { ControlMessage receivedCtrl = (ControlMessage) subgraphMappingInfoMessage.getControlInfo(); Integer partitionID = receivedCtrl.getPartitionID(); for (BytesWritable rawSubgraphID : receivedCtrl.getExtraInfo()) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(rawSubgraphID.copyBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.FullInfoSplitReaderInt.java
License:Apache License
private void createVertex(String stringInput) { // belongs to this partition String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[1])); int partitionID = Integer.parseInt(vertexValue[0]) - 1; LongWritable vertexSubgraphID = new LongWritable(Long.parseLong(vertexValue[2])); Subgraph<S, V, E, LongWritable, IntWritable, LongWritable> subgraph = (Subgraph<S, V, E, LongWritable, IntWritable, LongWritable>) partition .getSubgraph(vertexSubgraphID); if (subgraph == null) { subgraph = new Subgraph<S, V, E, LongWritable, IntWritable, LongWritable>(partitionID, vertexSubgraphID);// w w w . j a v a2s . c o m partition.addSubgraph(subgraph); } List<IEdge<E, LongWritable, IntWritable>> _adjList = new ArrayList<IEdge<E, LongWritable, IntWritable>>(); for (int j = 3; j < vertexValue.length; j++) { if (j + 3 > vertexValue.length) { LOG.debug("Incorrect length of line for vertex " + vertexID); } LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable sinkSubgraphID = new LongWritable(Long.parseLong(vertexValue[j + 1])); int sinkPartitionID = Integer.parseInt(vertexValue[j + 2]); j += 2; IntWritable edgeID = new IntWritable(edgeCount++ | ((peer.getPeerIndex()) << 27)); Edge<E, LongWritable, IntWritable> e = new Edge<E, LongWritable, IntWritable>(edgeID, sinkID); _adjList.add(e); if (sinkPartitionID != peer.getPeerIndex() && subgraph.getVertexById(sinkID) == null) { // this is a remote vertex IRemoteVertex<V, E, LongWritable, IntWritable, LongWritable> sink = new RemoteVertex<>(sinkID, sinkSubgraphID); // Add it to the same subgraph, as this is part of weakly connected // component subgraph.addVertex(sink); } } subgraph.addVertex(createVertexInstance(vertexID, _adjList)); }
From source file:in.dream_lab.goffish.hama.LongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { KeyValuePair<Writable, Writable> pair; long edgeCount = 0; vertexMap = Maps.newHashMap();//www. j a v a 2 s. c o m remoteVertexMap = Maps.newHashMap(); while ((pair = peer.readNext()) != null) { String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>( vertexID); for (int j = 1; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); } vertexMap.put(vertexID.get(), vertex); } /* Create remote vertex objects. */ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> e : vertex.getOutEdges()) { LongWritable sinkID = e.getSinkVertexId(); if (!vertexMap.containsKey(sinkID.get())) { IRemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = new RemoteVertex<>(sinkID); remoteVertexMap.put(sinkID.get(), sink); } } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<>( peer.getPeerIndex()); formSubgraphs(partition); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : remoteVertexMap.values()) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) remoteVertexMap .get(sinkID.get()); assert (sink != null); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.LongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition) throws IOException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertexMap.size() + remoteVertexMap.size()); for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { ds.addSet(vertex);/*from w w w . ja v a 2s . c om*/ } for (IVertex<V, E, LongWritable, LongWritable> vertex : remoteVertexMap.values()) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexId().get()); if (sink == null) { sink = remoteVertexMap.get(edge.getSinkVertexId().get()); } ds.union(vertex, sink); } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionId()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex // subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexId(), subgraph.getSubgraphId()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.hama.LongTextJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { // Map of partitionID,vertex that do not belong to this partition Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); JSONArray JSONInput = (JSONArray) JSONValue.parse(StringJSONInput); int partitionID = Integer.parseInt(JSONInput.get(1).toString()); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/*from w ww . ja va 2 s .co m*/ partitionVertices.add(StringJSONInput); } else { Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } //End of first SuperStep peer.sync(); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { String JSONVertex = msg.getControlInfo().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(JSONVertex); vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexId(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.LongTextJSONReader.java
License:Apache License
@SuppressWarnings("unchecked") Vertex<V, E, LongWritable, LongWritable> createVertex(String JSONString) { JSONArray JSONInput = (JSONArray) JSONValue.parse(JSONString); LongWritable sourceID = new LongWritable(Long.valueOf(JSONInput.get(0).toString())); assert (vertexMap.get(sourceID) == null); Vertex<V, E, LongWritable, LongWritable> vertex = new Vertex<V, E, LongWritable, LongWritable>(sourceID); //fix this// w w w. jav a2 s . c o m V value = (V) new Text(JSONInput.get(2).toString()); vertex.setValue(value); JSONArray edgeList = (JSONArray) JSONInput.get(3); for (Object edgeInfo : edgeList) { Object edgeValues[] = ((JSONArray) edgeInfo).toArray(); LongWritable sinkID = new LongWritable(Long.valueOf(edgeValues[0].toString())); LongWritable edgeID = new LongWritable(Long.valueOf(edgeValues[1].toString())); //fix this E edgeValue = (E) new Text(edgeValues[2].toString()); Edge<E, LongWritable, LongWritable> edge = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); edge.setValue(edgeValue); vertex.addEdge(edge); } return vertex; }
From source file:in.dream_lab.goffish.hama.LongTextJSONReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition, Collection<IVertex<V, E, LongWritable, LongWritable>> vertices) throws IOException { long subgraphCount = 0; Set<LongWritable> visited = new HashSet<LongWritable>(); Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(IMessage.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertices.size());/*from www . j av a 2s. c om*/ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertices) { if (!vertex.isRemote()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexId()); ds.union(vertex, sink); } } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionId()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexId(), subgraph.getSubgraphId()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }