List of usage examples for org.apache.hadoop.io LongWritable LongWritable
public LongWritable(long value)
From source file:in.dream_lab.goffish.giraph.examples.SingleSourceShortestPath.java
License:Apache License
/*** * MAIN COMPUTE METHOD/*from w w w . j a v a2 s . com*/ */ @Override public void compute(Iterable<IMessage<LongWritable, BytesWritable>> subgraphMessages) throws IOException { // long subgraphStartTime = System.currentTimeMillis(); ISubgraph<ShortestPathSubgraphValue, LongWritable, NullWritable, LongWritable, NullWritable, LongWritable> subgraph = getSubgraph(); // init IDs for logging // FIXME: Charith, we need an init() method later on // if(getSuperstep() == 0) { // partitionId = partition.getVertexId(); // subgraphId = subgraph.getVertexId(); // logFileName = "SP_" + partitionId + "_" + subgraphId + ".log"; // } // log("START superstep with received input messages count = " + packedSubGraphMessages.size()); Set<IVertex<LongWritable, NullWritable, LongWritable, NullWritable>> rootVertices = null; /////////////////////////////////////////////////////////// // First superstep. Get source superstep as input. // Initialize distances. calculate shortest distances in subgraph. if (getSuperstep() == 0) { // get input variables from init message // if(packedSubGraphMessages.size() == 0) { // throw new RuntimeException("Initial subgraph message was missing! Require sourceVertexID to be passed"); // } long sourceVertexID = Long.parseLong(getConf(SUBGRAPH_SOURCE_VERTEX)); // log("Initializing source vertex = " + sourceVertexID); // Giraph:SimpleShortestPathsComputation.java:64 // vertex.setValue(new DoubleWritable(Double.MAX_VALUE)); // initialize distance map of vertices to infinity // Note that if it is a remote vertex, we only have an estimate of the distance ShortestPathSubgraphValue subgraphValue = new ShortestPathSubgraphValue(); subgraph.setSubgraphValue(subgraphValue); subgraphValue.shortestDistanceMap = new HashMap<Long, Short>((int) subgraph.getVertexCount()); for (IVertex<LongWritable, NullWritable, LongWritable, NullWritable> v : subgraph.getLocalVertices()) { subgraphValue.shortestDistanceMap.put(v.getVertexId().get(), Short.MAX_VALUE); } for (IVertex<LongWritable, NullWritable, LongWritable, NullWritable> v : subgraph.getRemoteVertices()) { subgraphValue.shortestDistanceMap.put(v.getVertexId().get(), Short.MAX_VALUE); // remoteVertexCount++; } // Giraph:SimpleShortestPathsComputation.java:66 // double minDist = isSource(vertex) ? 0d : Double.MAX_VALUE; // Update distance to source as 0 boolean subgraphHasSource = false; if (subgraphValue.shortestDistanceMap.containsKey(sourceVertexID) && !subgraph.getVertexById(new LongWritable(sourceVertexID)).isRemote()) { subgraphValue.shortestDistanceMap.put(sourceVertexID, (short) 0); subgraphHasSource = true; } // If we have the source... if (subgraphHasSource) { IVertex sourceVertex = subgraph.getVertexById(new LongWritable(sourceVertexID)); rootVertices = new HashSet<>(1); rootVertices.add(sourceVertex); LOG.info("SourceVertex,Subgraph:" + sourceVertex + "," + subgraph.getSubgraphId()); } } else { /////////////////////////////////////////////////////////// // second superstep. int size = 0; for (IMessage<LongWritable, BytesWritable> iMessage : subgraphMessages) { size += iMessage.getMessage().getLength(); } // min(getLocalVertices, messsage length / ((8+2)/2)) rootVertices = new HashSet<>(Math.min((int) subgraph.getLocalVertexCount(), size / 5)); unpackSubgraphMessages(subgraphMessages, subgraph, rootVertices); // log("Unpacked messages count = " + subGraphMessages.size()); // // // We expect no more unique vertices than the number of input messages, // // or the total number of vertices. Note that we are likely over allocating. // // For directed graphs, it is not easy to find the number of in-boundary vertices. // rootVertices = new HashSet<>(Math.min(subGraphMessages.size(), (int) subgraph.getSubgraphVertices().getNumVertices())); // // // Giraph:SimpleShortestPathsComputation.java:68 // // minDist = Math.min(minDist, message.get()); // // // parse messages // // update distance map using messages if it has improved // // add the *unique set* of improved vertices to traversal list // for (String message : subGraphMessages) { // String[] tokens = message.split(","); // if(tokens.length != 2) { // throw new RuntimeException("Intermediate subgraph message did not contain 2 tokens. Has " + tokens.length + "instead"); // } // long sinkVertex = Long.parseLong(tokens[0]); // short sinkDistance = Short.parseShort(tokens[1]); // short distance = shortestDistanceMap.get(sinkVertex); // if(distance > sinkDistance){ // // path from remote is better than locally known path // shortestDistanceMap.put(sinkVertex, sinkDistance); // rootVertices.add(subgraph.getSubgraphVertices().getVertexById(new LongWritable(sinkVertex)); // } // } } ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphValue(); // Giraph:SimpleShortestPathsComputation.java:74 // if (minDist < vertex.getValue().get()) { // vertex.setValue(new DoubleWritable(minDist)); // for (Edge<LongWritable, FloatWritable> edge : vertex.getEdges()) { // double distance = minDist + edge.getValue().get(); // // if there are changes, then run dikstras if (rootVertices != null && rootVertices.size() > 0) { // List of remote vertices which could be affected by changes to distance // This does local agg that eliminates sending min dist to same vertex from // multiple vertices in this SG Set<Long> remoteUpdateSet = new HashSet<Long>( (int) (subgraph.getVertexCount() - subgraph.getLocalVertexCount())); // Update distances within local subgraph // Get list of remote vertices that were reached and updated. aStar(rootVertices, subgraphValue.shortestDistanceMap, remoteUpdateSet, subgraph); // log("END diskstras with subgraph local vertices="+ // (subgraph.getSubgraphVertices().getNumVertices() - remoteVertexCount) + "," +logMsg); // Giraph:SimpleShortestPathsComputation.java:82 // sendMessage(edge.getTargetVertexId(), new DoubleWritable(distance)); // Notify remote vertices of new known shortest distance from this subgraph and parent. // for(Long remoteVertexID : remoteUpdateSet){ // String payload = remoteVertexID + "," + shortestDistanceMap.get(remoteVertexID).toString(); // SubGraphMessage msg = new SubGraphMessage(payload.getBytes()); // msg.setTargetSubgraph(subgraph.getVertex(remoteVertexID).getRemoteSubgraphId()); // sendMessage(msg); // changeCount++; // } // Aggregate messages to remote subgraph packAndSendMessages(remoteUpdateSet, subgraph); } // log("END superstep. Sent remote vertices = " + changeCount + ", remote messages =" + messageCount); // if no distances were changed, we terminate. // if no one's distances change, everyone has votd to halt // if(changeCount == 0) { // we're done voteToHalt(); // } long subgraphEndTime = System.currentTimeMillis(); // logPerfString("SUBGRAPH_PERF ,"+subgraph.getVertexId() +" ," + getSuperStep() + " ," +getIteration() + " ,"+ subgraphStartTime // + " ,"+subgraphEndTime + " ," + (subgraphEndTime - subgraphStartTime)+ " ,"+subgraph.numVertices() + " ," + subgraph.numEdges()); }
From source file:in.dream_lab.goffish.giraph.examples.SingleSourceShortestPath.java
License:Apache License
void packAndSendMessages(Set<Long> remoteVertexUpdates, ISubgraph<ShortestPathSubgraphValue, LongWritable, NullWritable, LongWritable, NullWritable, LongWritable> subgraph) throws IOException { ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphValue(); HashMap<LongWritable, ExtendedByteArrayDataOutput> messagesMap = new HashMap<>(); int unpackedMessageCount = 0; for (long entry : remoteVertexUpdates) { unpackedMessageCount++;/*from w ww .ja va 2s .c om*/ IRemoteVertex<LongWritable, NullWritable, LongWritable, NullWritable, LongWritable> remoteSubgraphVertex = (IRemoteVertex) subgraph .getVertexById(new LongWritable(entry)); ExtendedByteArrayDataOutput dataOutput; if (!messagesMap.containsKey(remoteSubgraphVertex.getSubgraphId())) { dataOutput = new ExtendedByteArrayDataOutput(); messagesMap.put(remoteSubgraphVertex.getSubgraphId(), dataOutput); } else { dataOutput = messagesMap.get(remoteSubgraphVertex.getSubgraphId()); } dataOutput.writeLong(remoteSubgraphVertex.getVertexId().get()); dataOutput.writeShort(subgraphValue.shortestDistanceMap.get(entry)); } for (Map.Entry<LongWritable, ExtendedByteArrayDataOutput> entry : messagesMap.entrySet()) { ExtendedByteArrayDataOutput dataOutput = entry.getValue(); dataOutput.writeLong(-1); sendMessage(entry.getKey(), new BytesWritable(dataOutput.getByteArray())); } LOG.info("Superstep,SubgraphId,unpackedSentMessageCount,packedSentMessageCount:" + getSuperstep() + "," + subgraph.getSubgraphId() + "," + unpackedMessageCount + "," + messagesMap.size()); }
From source file:in.dream_lab.goffish.giraph.examples.SingleSourceShortestPath.java
License:Apache License
private void unpackSubgraphMessages(Iterable<IMessage<LongWritable, BytesWritable>> packedSubGraphMessages, ISubgraph<ShortestPathSubgraphValue, LongWritable, NullWritable, LongWritable, NullWritable, LongWritable> subgraph, Set<IVertex<LongWritable, NullWritable, LongWritable, NullWritable>> rootVertices) throws IOException { ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphValue(); int unpackedMessageCount = 0; int packedMessageCount = 0; for (IMessage<LongWritable, BytesWritable> iMessage : packedSubGraphMessages) { packedMessageCount++;/*from w ww. ja v a2 s. c om*/ BytesWritable subgraphMessageValue = iMessage.getMessage(); ExtendedByteArrayDataInput dataInput = new ExtendedByteArrayDataInput(subgraphMessageValue.getBytes()); while (!dataInput.endOfInput()) { long sinkVertex = dataInput.readLong(); if (sinkVertex == -1) { break; } unpackedMessageCount++; short sinkDistance = dataInput.readShort(); //LOG.info("Test, Sink vertex received: " + sinkVertex); // SubgraphVertex<LongWritable, LongWritable, LongWritable, NullWritable, NullWritable> currentVertex = vertices.get(new LongWritable(sinkVertex)); //LOG.info("Test, Current vertex object: " + currentVertex); //LOG.info("Test, Current vertex: " + currentVertex.getVertexId()); short distance = subgraphValue.shortestDistanceMap.get(sinkVertex); if (sinkDistance < distance) { subgraphValue.shortestDistanceMap.put(sinkVertex, sinkDistance); rootVertices.add(subgraph.getVertexById(new LongWritable(sinkVertex))); } } } LOG.info("Superstep,SubgraphId,unpackedReceivedMessageCount,packedReceivedMessageCount,rootVertices:" + getSuperstep() + "," + subgraph.getSubgraphId() + "," + unpackedMessageCount + "," + packedMessageCount + "," + rootVertices.size()); }
From source file:in.dream_lab.goffish.giraph.examples.SubgraphSingleSourceShortestPathWithWeights.java
License:Apache License
/*** * MAIN COMPUTE METHOD/*from w w w . ja va 2 s . c om*/ */ @Override public void compute(Iterable<IMessage<LongWritable, BytesWritable>> subgraphMessages) throws IOException { // long subgraphStartTime = System.currentTimeMillis(); ISubgraph<ShortestPathSubgraphValue, LongWritable, DoubleWritable, LongWritable, NullWritable, LongWritable> subgraph = getSubgraph(); try { // init IDs for logging // FIXME: Charith, we need an init() method later on // if(getSuperstep() == 0) { // partitionId = partition.getVertexId(); // subgraphId = subgraph.getVertexId(); // logFileName = "SP_" + partitionId + "_" + subgraphId + ".log"; // } // log("START superstep with received input messages count = " + packedSubGraphMessages.size()); Set<IVertex<LongWritable, DoubleWritable, LongWritable, NullWritable>> rootVertices = null; /////////////////////////////////////////////////////////// // First superstep. Get source superstep as input. // Initialize distances. calculate shortest distances in subgraph. if (getSuperstep() == 0) { // get input variables from init message // if(packedSubGraphMessages.size() == 0) { // throw new RuntimeException("Initial subgraph message was missing! Require sourceVertexID to be passed"); // } long sourceVertexID = Long.parseLong(getConf(SUBGRAPH_SOURCE_VERTEX)); // log("Initializing source vertex = " + sourceVertexID); // Giraph:SimpleShortestPathsComputation.java:64 // vertex.setValue(new DoubleWritable(Double.MAX_VALUE)); // initialize distance map of vertices to infinity // Note that if it is a remote vertex, we only have an estimate of the distance ShortestPathSubgraphValue subgraphValue = new ShortestPathSubgraphValue(); subgraph.setSubgraphValue(subgraphValue); subgraphValue.shortestDistanceMap = new HashMap<Long, Short>((int) subgraph.getLocalVertexCount()); for (IVertex<LongWritable, DoubleWritable, LongWritable, NullWritable> v : subgraph .getLocalVertices()) { subgraphValue.shortestDistanceMap.put(v.getVertexId().get(), Short.MAX_VALUE); } for (IVertex<LongWritable, DoubleWritable, LongWritable, NullWritable> v : subgraph .getRemoteVertices()) { subgraphValue.shortestDistanceMap.put(v.getVertexId().get(), Short.MAX_VALUE); // remoteVertexCount++; } // Giraph:SimpleShortestPathsComputation.java:66 // double minDist = isSource(vertex) ? 0d : Double.MAX_VALUE; // Update distance to source as 0 boolean subgraphHasSource = false; if (subgraphValue.shortestDistanceMap.containsKey(sourceVertexID) && !subgraph.getVertexById(new LongWritable(sourceVertexID)).isRemote()) { subgraphValue.shortestDistanceMap.put(sourceVertexID, (short) 0); subgraphHasSource = true; } // If we have the source... if (subgraphHasSource) { IVertex sourceVertex = subgraph.getVertexById(new LongWritable(sourceVertexID)); rootVertices = new HashSet<>(1); rootVertices.add(sourceVertex); } } else { /////////////////////////////////////////////////////////// // second superstep. int size = 0; for (IMessage<LongWritable, BytesWritable> iMessage : subgraphMessages) { size += iMessage.getMessage().getLength(); } // min(getLocalVertices, messsage length / ((8+2)/2)) rootVertices = new HashSet<>(Math.min((int) subgraph.getLocalVertexCount(), size / 5)); unpackSubgraphMessages(subgraphMessages, subgraph, rootVertices); // log("Unpacked messages count = " + subGraphMessages.size()); // // // We expect no more unique vertices than the number of input messages, // // or the total number of vertices. Note that we are likely over allocating. // // For directed graphs, it is not easy to find the number of in-boundary vertices. // rootVertices = new HashSet<>(Math.min(subGraphMessages.size(), (int) subgraph.getSubgraphVertices().getNumVertices())); // // // Giraph:SimpleShortestPathsComputation.java:68 // // minDist = Math.min(minDist, message.get()); // // // parse messages // // update distance map using messages if it has improved // // add the *unique set* of improved vertices to traversal list // for (String message : subGraphMessages) { // String[] tokens = message.split(","); // if(tokens.length != 2) { // throw new RuntimeException("Intermediate subgraph message did not contain 2 tokens. Has " + tokens.length + "instead"); // } // long sinkVertex = Long.parseLong(tokens[0]); // short sinkDistance = Short.parseShort(tokens[1]); // short distance = shortestDistanceMap.get(sinkVertex); // if(distance > sinkDistance){ // // path from remote is better than locally known path // shortestDistanceMap.put(sinkVertex, sinkDistance); // rootVertices.add(subgraph.getSubgraphVertices().getVertexById(new LongWritable(sinkVertex)); // } // } } ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphValue(); // Giraph:SimpleShortestPathsComputation.java:74 // if (minDist < vertex.getValue().get()) { // vertex.setValue(new DoubleWritable(minDist)); // for (Edge<LongWritable, FloatWritable> edge : vertex.getEdges()) { // double distance = minDist + edge.getValue().get(); // // if there are changes, then run dikstras int changeCount = 0; int messageCount = 0; if (rootVertices != null && rootVertices.size() > 0) { // List of remote vertices which could be affected by changes to distance // This does local agg that eliminates sending min dist to same vertex from // multiple vertices in this SG Set<Long> remoteUpdateSet = new HashSet<Long>( (int) (subgraph.getVertexCount() - subgraph.getLocalVertexCount())); // Update distances within local subgraph // Get list of remote vertices that were reached and updated. String logMsg = aStar(rootVertices, subgraphValue.shortestDistanceMap, remoteUpdateSet, subgraph); // log("END diskstras with subgraph local vertices="+ // (subgraph.getSubgraphVertices().getNumVertices() - remoteVertexCount) + "," +logMsg); // Giraph:SimpleShortestPathsComputation.java:82 // sendMessage(edge.getTargetVertexId(), new DoubleWritable(distance)); // Notify remote vertices of new known shortest distance from this subgraph and parent. // for(Long remoteVertexID : remoteUpdateSet){ // String payload = remoteVertexID + "," + shortestDistanceMap.get(remoteVertexID).toString(); // SubGraphMessage msg = new SubGraphMessage(payload.getBytes()); // msg.setTargetSubgraph(subgraph.getVertex(remoteVertexID).getRemoteSubgraphId()); // sendMessage(msg); // changeCount++; // } // Aggregate messages to remote subgraph changeCount = remoteUpdateSet.size(); messageCount = packAndSendMessages(remoteUpdateSet, subgraph); } // if no distances were changed, we terminate. // if no one's distances change, everyone has votd to halt // if(changeCount == 0) { // we're done voteToHalt(); // } } catch (RuntimeException ex) { // if(logFileName == null) logFileName = "ERROR.log"; // log("Unknown error in compute", ex); } long subgraphEndTime = System.currentTimeMillis(); // logPerfString("SUBGRAPH_PERF ,"+subgraph.getVertexId() +" ," + getSuperStep() + " ," +getIteration() + " ,"+ subgraphStartTime // + " ,"+subgraphEndTime + " ," + (subgraphEndTime - subgraphStartTime)+ " ,"+subgraph.numVertices() + " ," + subgraph.numEdges()); }
From source file:in.dream_lab.goffish.giraph.examples.SubgraphSingleSourceShortestPathWithWeights.java
License:Apache License
int packAndSendMessages(Set<Long> remoteVertexUpdates, ISubgraph<ShortestPathSubgraphValue, LongWritable, DoubleWritable, LongWritable, NullWritable, LongWritable> subgraph) throws IOException { ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphValue(); HashMap<LongWritable, ExtendedByteArrayDataOutput> messagesMap = new HashMap<>(); for (long entry : remoteVertexUpdates) { IRemoteVertex<LongWritable, NullWritable, LongWritable, NullWritable, LongWritable> remoteSubgraphVertex = (IRemoteVertex) subgraph .getVertexById(new LongWritable(entry)); ExtendedByteArrayDataOutput dataOutput; if (!messagesMap.containsKey(remoteSubgraphVertex.getSubgraphId())) { dataOutput = new ExtendedByteArrayDataOutput(); messagesMap.put(remoteSubgraphVertex.getSubgraphId(), dataOutput); } else {//w w w .j a va 2 s. co m dataOutput = messagesMap.get(remoteSubgraphVertex.getSubgraphId()); } dataOutput.writeLong(remoteSubgraphVertex.getVertexId().get()); dataOutput.writeShort(subgraphValue.shortestDistanceMap.get(entry)); // LOG.info("SubgraphID" + remoteSubgraphVertex.getSubgraphId() + " Sending vertex id " + remoteSubgraphVertex.getVertexId().get() + " distance "+ entry.getValue()); } int messageCount = 0; for (Map.Entry<LongWritable, ExtendedByteArrayDataOutput> entry : messagesMap.entrySet()) { ExtendedByteArrayDataOutput dataOutput = entry.getValue(); dataOutput.writeLong(-1); sendMessage(entry.getKey(), new BytesWritable(dataOutput.getByteArray())); messageCount++; } return messageCount; }
From source file:in.dream_lab.goffish.giraph.examples.SubgraphSingleSourceShortestPathWithWeights.java
License:Apache License
private void unpackSubgraphMessages(Iterable<IMessage<LongWritable, BytesWritable>> packedSubGraphMessages, ISubgraph<ShortestPathSubgraphValue, LongWritable, DoubleWritable, LongWritable, NullWritable, LongWritable> subgraph, Set<IVertex<LongWritable, DoubleWritable, LongWritable, NullWritable>> rootVertices) throws IOException { ShortestPathSubgraphValue subgraphValue = subgraph.getSubgraphValue(); for (IMessage<LongWritable, BytesWritable> iMessage : packedSubGraphMessages) { BytesWritable subgraphMessageValue = iMessage.getMessage(); ExtendedByteArrayDataInput dataInput = new ExtendedByteArrayDataInput(subgraphMessageValue.getBytes()); while (!dataInput.endOfInput()) { long sinkVertex = dataInput.readLong(); if (sinkVertex == -1) { break; }// w ww . j av a2s . c o m short sinkDistance = dataInput.readShort(); //LOG.info("Test, Sink vertex received: " + sinkVertex); // SubgraphVertex<LongWritable, LongWritable, LongWritable, NullWritable, NullWritable> currentVertex = vertices.get(new LongWritable(sinkVertex)); //LOG.info("Test, Current vertex object: " + currentVertex); //LOG.info("Test, Current vertex: " + currentVertex.getVertexId()); short distance = subgraphValue.shortestDistanceMap.get(sinkVertex); if (sinkDistance < distance) { subgraphValue.shortestDistanceMap.put(sinkVertex, sinkDistance); rootVertices.add(subgraph.getVertexById(new LongWritable(sinkVertex))); } } } }
From source file:in.dream_lab.goffish.giraph.examples.SubgraphTriangleCount.java
License:Apache License
@Override public void compute(Iterable<IMessage<LongWritable, BytesWritable>> subgraphMessages) throws IOException { // Convert adjacency list to adjacency set ISubgraph<TriangleCountSubgraphValue, LongWritable, NullWritable, LongWritable, LongWritable, LongWritable> subgraph = getSubgraph(); if (getSuperstep() == 0) { TriangleCountSubgraphValue triangleCountSubgraphValue = new TriangleCountSubgraphValue(); triangleCountSubgraphValue.adjSet = new HashMap<Long, Set<Long>>(); subgraph.setSubgraphValue(triangleCountSubgraphValue); for (IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex : subgraph .getLocalVertices()) {// ww w.j a v a2s. co m Set<Long> adjVertices = new HashSet<Long>(); for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { adjVertices.add(edge.getSinkVertexId().get()); } triangleCountSubgraphValue.adjSet.put(vertex.getVertexId().get(), adjVertices); } return; } else if (getSuperstep() == 1) { TriangleCountSubgraphValue triangleCountSubgraphValue = subgraph.getSubgraphValue(); long triangleCount = triangleCountSubgraphValue.triangleCount; Map<LongWritable, ExtendedByteArrayDataOutput> msg = new HashMap<>(); for (IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex : subgraph .getLocalVertices()) { for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> adjVertex = subgraph .getVertexById(edge.getSinkVertexId()); // Preparing messages to be sent to remote adjacent vertices. if (adjVertex.isRemote() && adjVertex.getVertexId().get() > vertex.getVertexId().get()) { LongWritable remoteSubgraphId = ((IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable>) adjVertex) .getSubgraphId(); ExtendedByteArrayDataOutput vertexIds = msg.get(remoteSubgraphId); if (vertexIds == null) { vertexIds = new ExtendedByteArrayDataOutput(); msg.put(remoteSubgraphId, vertexIds); } vertexIds.writeLong(adjVertex.getVertexId().get()); vertexIds.writeLong(vertex.getVertexId().get()); vertexIds.writeLong(vertex.getVertexId().get()); } else if (adjVertex.isRemote() || vertex.getVertexId().get() > adjVertex.getVertexId().get()) continue; if (adjVertex.isRemote()) { continue; //as it has no outedges } // Counting triangles which have at least two vertices in the same // subgraph. for (IEdge<NullWritable, LongWritable, LongWritable> edgeAdjVertex : adjVertex.getOutEdges()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> adjAdjVertex = subgraph .getVertexById(edgeAdjVertex.getSinkVertexId()); if (adjAdjVertex.isRemote() || adjAdjVertex.getVertexId().get() > adjVertex.getVertexId().get()) { if (triangleCountSubgraphValue.adjSet.get(vertex.getVertexId().get()) .contains(adjAdjVertex.getVertexId().get())) { triangleCount++; //trianglesList.append(vertex.getVertexID().get() + " " + adjVertex.getVertexID().get() // + " " + adjAdjVertex.getVertexID().get() + "\n"); } } } } } triangleCountSubgraphValue.triangleCount = triangleCount; sendPackedMessages(msg); } else if (getSuperstep() == 2) { Map<Long, List<Pair<Long, Long>>> ids = new HashMap<Long, List<Pair<Long, Long>>>(); unpackMessages(subgraphMessages, ids); Map<LongWritable, ExtendedByteArrayDataOutput> msg = new HashMap<>(); for (Map.Entry<Long, List<Pair<Long, Long>>> entry : ids.entrySet()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex = subgraph .getVertexById(new LongWritable(entry.getKey())); List<Pair<Long, Long>> idPairs = entry.getValue(); for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> adjVertex = subgraph .getVertexById(edge.getSinkVertexId()); if (adjVertex.isRemote() && adjVertex.getVertexId().get() > vertex.getVertexId().get()) { LongWritable remoteSubgraphId = ((IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable>) adjVertex) .getSubgraphId(); ExtendedByteArrayDataOutput vertexIds = msg.get(remoteSubgraphId); if (vertexIds == null) { vertexIds = new ExtendedByteArrayDataOutput(); msg.put(remoteSubgraphId, vertexIds); } for (Pair<Long, Long> id : idPairs) { LongWritable firstId = new LongWritable(id.first); IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable> sinkSubgraphID = (IRemoteVertex<LongWritable, NullWritable, LongWritable, LongWritable, LongWritable>) subgraph .getVertexById(firstId); if (sinkSubgraphID.getSubgraphId() != remoteSubgraphId) { vertexIds.writeLong(adjVertex.getVertexId().get()); vertexIds.writeLong(firstId.get()); vertexIds.writeLong(vertex.getVertexId().get()); } } } } } sendPackedMessages(msg); } else { long triangleCount = subgraph.getSubgraphValue().triangleCount; Map<Long, List<Pair<Long, Long>>> ids = new HashMap<Long, List<Pair<Long, Long>>>(); unpackMessages(subgraphMessages, ids); for (Map.Entry<Long, List<Pair<Long, Long>>> entry : ids.entrySet()) { IVertex<LongWritable, NullWritable, LongWritable, LongWritable> vertex = subgraph .getVertexById(new LongWritable(entry.getKey())); for (Pair<Long, Long> p : entry.getValue()) { for (IEdge<NullWritable, LongWritable, LongWritable> edge : vertex.getOutEdges()) { if (edge.getSinkVertexId().get() == p.first) { triangleCount++; } } } } subgraph.getSubgraphValue().triangleCount = triangleCount; } voteToHalt(); }
From source file:in.dream_lab.goffish.hama.DenseGraphLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { /* Used for logging */ Runtime runtime = Runtime.getRuntime(); int mb = 1024 * 1024; LOG.info("Free Memory in Reader: " + runtime.freeMemory() / mb + " Total Memory: " + runtime.totalMemory() / mb); LOG.info("Free Memory after Reaching reader " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; long edgeCount = 0; vertexMap = Maps.newHashMap();//from w ww . ja v a2 s . c o m remoteVertexMap = Maps.newHashMap(); LOG.info("SETUP Starting Free Memory: " + runtime.freeMemory() / mb + " Total Memory: " + runtime.totalMemory() / mb); LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); int count = -1; while ((pair = peer.readNext()) != null) { count++; // NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); List<IEdge<E, LongWritable, LongWritable>> _adjList = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); for (int j = 1; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); _adjList.add(e); } vertexMap.put(vertexID.get(), createVertexInstance(vertexID, _adjList)); } LOG.info("Number of Vertices: " + vertexMap.size() + " Edges: " + edgeCount); LOG.info("Free Memory: " + runtime.freeMemory() / mb + " Total Memory: " + runtime.totalMemory() / mb); System.gc(); peer.sync(); LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> e : vertex.getOutEdges()) { LongWritable sinkID = e.getSinkVertexId(); if (!vertexMap.containsKey(sinkID.get())) { IRemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = new RemoteVertex<>(sinkID); remoteVertexMap.put(sinkID.get(), sink); } } } peer.sync(); Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition); //clearing used memory vertexMap = null; LOG.info("Done with formSubgraph()"); /* * Tell other partitions our Vertex Ids and their subgraphIDs */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: subgraphID1 count1 vertex1 vertex2 ... subgraphID2 count2 vertex1 vertex2 ... */ for (ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraphs : partition.getSubgraphs()) { controlInfo.addextraInfo(Longs.toByteArray(subgraphs.getSubgraphId().get())); controlInfo.addextraInfo(Longs.toByteArray(subgraphs.getLocalVertexCount())); for (IVertex<V, E, LongWritable, LongWritable> v : subgraphs.getLocalVertices()) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); LOG.info("Completed first superstep in reader"); System.out.println("Before 2nd Superstep " + (runtime.totalMemory() - runtime.freeMemory()) / mb); peer.sync(); LOG.info("Started superstep 2 in reader"); Message<LongWritable, LongWritable> msg; Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving vertices to set Remote Vertex subgraph id. */ Iterator<BytesWritable> remoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo() .iterator(); while (remoteVertexQuery.hasNext()) { Long subgraphID = Longs.fromByteArray(remoteVertexQuery.next().getBytes()); Long vertexCount = Longs.fromByteArray(remoteVertexQuery.next().getBytes()); for (long i = 0; i < vertexCount; i++) { Long remoteVertexID = Longs.fromByteArray(remoteVertexQuery.next().getBytes()); if (remoteVertexMap.containsKey(remoteVertexID)) { RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) remoteVertexMap .get(remoteVertexID); sink.setSubgraphID(new LongWritable(subgraphID)); } } } } LOG.info("Completed 2nd superstep in reader"); LOG.info("Reader finished"); return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.DenseGraphLongTextAdjacencyListReader.java
License:Apache License
void formSubgraphs(Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition) throws IOException, SyncException, InterruptedException { long subgraphCount = 0; Message<LongWritable, LongWritable> subgraphLocationBroadcast = new Message<LongWritable, LongWritable>(); subgraphLocationBroadcast.setMessageType(Message.MessageType.SUBGRAPH); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); subgraphLocationBroadcast.setControlInfo(controlInfo); byte partitionBytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionBytes); // initialize disjoint set DisjointSets<IVertex<V, E, LongWritable, LongWritable>> ds = new DisjointSets<IVertex<V, E, LongWritable, LongWritable>>( vertexMap.size() + remoteVertexMap.size()); for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { ds.addSet(vertex);// www . j ava 2 s . com } for (IVertex<V, E, LongWritable, LongWritable> vertex : remoteVertexMap.values()) { ds.addSet(vertex); } // union edge pairs for (IVertex<V, E, LongWritable, LongWritable> vertex : vertexMap.values()) { for (IEdge<E, LongWritable, LongWritable> edge : vertex.getOutEdges()) { IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(edge.getSinkVertexId().get()); if (sink == null) { sink = remoteVertexMap.get(edge.getSinkVertexId().get()); } ds.union(vertex, sink); } } Collection<? extends Collection<IVertex<V, E, LongWritable, LongWritable>>> components = ds.retrieveSets(); for (Collection<IVertex<V, E, LongWritable, LongWritable>> component : components) { LongWritable subgraphID = new LongWritable( subgraphCount++ | (((long) partition.getPartitionId()) << 32)); Subgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph = new Subgraph<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex(), subgraphID); for (IVertex<V, E, LongWritable, LongWritable> vertex : component) { subgraph.addVertex(vertex); // Dont add remote vertices to the VertexSubgraphMap as remote vertex // subgraphID is unknown if (!vertex.isRemote()) { vertexSubgraphMap.put(vertex.getVertexId(), subgraph.getSubgraphId()); } } partition.addSubgraph(subgraph); byte subgraphIDbytes[] = Longs.toByteArray(subgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphLocationBroadcast); }
From source file:in.dream_lab.goffish.hama.FullInfoNonSplitReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { long startTime = System.currentTimeMillis(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String stringInput = pair.getValue().toString(); createVertex(stringInput);//from w w w . ja va 2s . c o m } LOG.debug("Finished Graph Loading in partition" + peer.getPeerIndex()); // broadcast all subgraphs belonging to this partition Message<K, M> subgraphMapppingMessage = new Message<>(); subgraphMapppingMessage.setMessageType(Message.MessageType.CUSTOM_MESSAGE); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); controlInfo.setPartitionID(peer.getPeerIndex()); subgraphMapppingMessage.setControlInfo(controlInfo); for (ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable> subgraph : partition.getSubgraphs()) { byte subgraphIDbytes[] = Longs.toByteArray(subgraph.getSubgraphId().get()); controlInfo.addextraInfo(subgraphIDbytes); } sendToAllPartitions(subgraphMapppingMessage); LOG.debug("Subgraph partition Broadcast sent"); long endTime = System.currentTimeMillis(); LOG.info("GOFFISH3.PERF.GRAPH_LOAD," + peer.getPeerIndex() + "," + peer.getSuperstepCount() + "," + startTime + "," + endTime + "," + (endTime - startTime)); peer.sync(); startTime = System.currentTimeMillis(); Message<K, M> subgraphMappingInfoMessage; while ((subgraphMappingInfoMessage = peer.getCurrentMessage()) != null) { ControlMessage receivedCtrl = (ControlMessage) subgraphMappingInfoMessage.getControlInfo(); Integer partitionID = receivedCtrl.getPartitionID(); for (BytesWritable rawSubgraphID : receivedCtrl.getExtraInfo()) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(rawSubgraphID.copyBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } } LOG.debug("Reader Completed"); endTime = System.currentTimeMillis(); LOG.info("GOFFISH3.PERF.GRAPH_LOAD," + peer.getPeerIndex() + "," + peer.getSuperstepCount() + "," + startTime + "," + endTime + "," + (endTime - startTime)); return partition.getSubgraphs(); }