List of usage examples for org.apache.hadoop.io BytesWritable getBytes
@Override public byte[] getBytes()
From source file:in.dream_lab.goffish.hama.LongTextJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { // Map of partitionID,vertex that do not belong to this partition Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); JSONArray JSONInput = (JSONArray) JSONValue.parse(StringJSONInput); int partitionID = Integer.parseInt(JSONInput.get(1).toString()); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/*w ww. j a va 2 s . com*/ partitionVertices.add(StringJSONInput); } else { Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } //End of first SuperStep peer.sync(); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { String JSONVertex = msg.getControlInfo().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(JSONVertex); vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexId(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.hama.PartitionsLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.debug("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { //NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); //LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); int partitionID = Integer.parseInt(vertexValue[1]); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }// www . j ava 2s . co m partitionVertices.add(stringInput); } else { LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 2; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); _edges.add(e); } vertexMap.put(vertex.getVertexId(), vertex); _edges.addAll(vertex.getOutEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } // End of first superstep. peer.sync(); LOG.debug("Second Superstep in Reader " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { ControlMessage ctrlMessage = (ControlMessage) msg.getControlInfo(); String msgString = ctrlMessage.getVertexValues(); // String msgStringArr[] = msgString.split(","); // for (int i = 0; i < msgStringArr.length; i++) { String vertexInfo[] = msgString.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexInfo[0])); Vertex<V, E, LongWritable, LongWritable> source = (Vertex<V, E, LongWritable, LongWritable>) vertexMap .get(vertexID); if (source == null) { source = new Vertex<V, E, LongWritable, LongWritable>(vertexID); vertexMap.put(source.getVertexId(), source); } for (int j = 2; j < vertexInfo.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexInfo[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); source.addEdge(e); _edges.add(e); } } //} LOG.debug("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexId(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.debug("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.debug("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexId().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); assert (sink != null); sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongMapJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { LOG.info("Creating vertices"); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); }/*from w ww . j a va 2s . co m*/ LOG.info("Sending Vertices to respective partitions"); LOG.info("Received all vertices"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { //NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); //LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); int partitionID = Integer.parseInt(vertexValue[1]); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }//from w w w . j a v a 2 s . c o m partitionVertices.add(stringInput); } else { LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 2; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e); _edges.add(e); } vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); System.out.println("To " + partitionID + " " + vertices.size()); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } // End of first superstep. System.gc(); peer.sync(); LOG.info("Second Superstep in Reader " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { ControlMessage ctrlMessage = (ControlMessage) msg.getControlInfo(); String msgString = ctrlMessage.getVertexValues(); // String msgStringArr[] = msgString.split(","); // for (int i = 0; i < msgStringArr.length; i++) { String vertexInfo[] = msgString.split("\\s+"); LongWritable vertexID = new LongWritable(Long.parseLong(vertexInfo[0])); Vertex<V, E, LongWritable, LongWritable> source = (Vertex<V, E, LongWritable, LongWritable>) vertexMap .get(vertexID); if (source == null) { source = new Vertex<V, E, LongWritable, LongWritable>(vertexID); vertexMap.put(source.getVertexID(), source); } for (int j = 2; j < vertexInfo.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexInfo[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); source.addEdge(e); _edges.add(e); } } //} LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.info("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.LongTextJSONReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { // Map of partitionID,vertex that do not belong to this partition Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { String StringJSONInput = pair.getValue().toString(); JSONArray JSONInput = (JSONArray) JSONValue.parse(StringJSONInput); int partitionID = Integer.parseInt(JSONInput.get(1).toString()); // Vertex does not belong to this partition if (partitionID != peer.getPeerIndex()) { List<String> partitionVertices = partitionMap.get(partitionID); if (partitionVertices == null) { partitionVertices = new ArrayList<String>(); partitionMap.put(partitionID, partitionVertices); }/*ww w. jav a 2 s .c o m*/ partitionVertices.add(StringJSONInput); } else { Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(StringJSONInput); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } } // Send vertices to their respective partitions for (Map.Entry<Integer, List<String>> entry : partitionMap.entrySet()) { int partitionID = entry.getKey().intValue(); List<String> vertices = entry.getValue(); for (String vertex : vertices) { Message<LongWritable, LongWritable> vertexMsg = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.VERTEX); controlInfo.setVertexValues(vertex); vertexMsg.setControlInfo(controlInfo); peer.send(peer.getPeerName(partitionID), (Message<K, M>) vertexMsg); } } //End of first SuperStep peer.sync(); Message<LongWritable, LongWritable> msg; while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { String JSONVertex = msg.getControlInfo().toString(); Vertex<V, E, LongWritable, LongWritable> vertex = createVertex(JSONVertex); vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } //Direct Copy paste from here Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); formSubgraphs(partition, vertexMap.values()); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: * partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); //Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast * Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : * sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); //In case this partition does not have the vertex /* Case 1 : If vertex does not exist * Case 2 : If vertex exists but is remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:in.dream_lab.goffish.PartitionsLongTextAdjacencyListReader.java
License:Apache License
@Override public List<ISubgraph<S, V, E, LongWritable, LongWritable, LongWritable>> getSubgraphs() throws IOException, SyncException, InterruptedException { Map<Integer, List<String>> partitionMap = new HashMap<Integer, List<String>>(); vertexMap = new HashMap<LongWritable, IVertex<V, E, LongWritable, LongWritable>>(); // List of edges.Used to create RemoteVertices List<IEdge<E, LongWritable, LongWritable>> _edges = new ArrayList<IEdge<E, LongWritable, LongWritable>>(); long edgeCount = 0; LOG.info("SETUP Starting " + peer.getPeerIndex() + " Memory: " + Runtime.getRuntime().freeMemory()); KeyValuePair<Writable, Writable> pair; while ((pair = peer.readNext()) != null) { // NOTE: Confirm that data starts from value and not from key. String stringInput = pair.getValue().toString(); String vertexValue[] = stringInput.split("\\s+"); // LongWritable sourceID = new LongWritable(Long.parseLong(value[0])); // int partitionID = Integer.parseInt(vertexValue[1]); LongWritable vertexID = new LongWritable(Long.parseLong(vertexValue[0])); Vertex<V, E, LongWritable, LongWritable> vertex; vertex = new Vertex<V, E, LongWritable, LongWritable>(vertexID); for (int j = 1; j < vertexValue.length; j++) { LongWritable sinkID = new LongWritable(Long.parseLong(vertexValue[j])); LongWritable edgeID = new LongWritable(edgeCount++ | (((long) peer.getPeerIndex()) << 32)); Edge<E, LongWritable, LongWritable> e = new Edge<E, LongWritable, LongWritable>(edgeID, sinkID); vertex.addEdge(e);//from w w w .j a v a2 s . co m _edges.add(e); } vertexMap.put(vertex.getVertexID(), vertex); _edges.addAll(vertex.outEdges()); } LOG.info("Number of Vertices: " + vertexMap.size() + " Edges: " + _edges.size()); LOG.info("Creating Remote Vertex Objects"); /* Create remote vertex objects. */ for (IEdge<E, LongWritable, LongWritable> e : _edges) { LongWritable sinkID = e.getSinkVertexID(); IVertex<V, E, LongWritable, LongWritable> sink = vertexMap.get(sinkID); if (sink == null) { sink = new RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>(sinkID); vertexMap.put(sinkID, sink); } } Partition<S, V, E, LongWritable, LongWritable, LongWritable> partition = new Partition<S, V, E, LongWritable, LongWritable, LongWritable>( peer.getPeerIndex()); LOG.info("Calling formSubgraph()"); formSubgraphs(partition, vertexMap.values()); LOG.info("Done with formSubgraph()"); /* * Ask Remote vertices to send their subgraph IDs. Requires 2 supersteps * because the graph is directed */ Message<LongWritable, LongWritable> question = new Message<LongWritable, LongWritable>(); ControlMessage controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.BROADCAST); question.setControlInfo(controlInfo); /* * Message format being sent: partitionID remotevertex1 remotevertex2 ... */ byte partitionIDbytes[] = Ints.toByteArray(peer.getPeerIndex()); controlInfo.addextraInfo(partitionIDbytes); for (IVertex<V, E, LongWritable, LongWritable> v : vertexMap.values()) { if (v instanceof RemoteVertex) { byte vertexIDbytes[] = Longs.toByteArray(v.getVertexID().get()); controlInfo.addextraInfo(vertexIDbytes); } } sendToAllPartitions(question); peer.sync(); Message<LongWritable, LongWritable> msg; Map<Integer, List<Message<LongWritable, LongWritable>>> replyMessages = new HashMap<Integer, List<Message<LongWritable, LongWritable>>>(); // Receiving 1 message per partition while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { /* * Subgraph Partition mapping broadcast Format of received message: * partitionID subgraphID1 subgraphID2 ... */ if (msg.getMessageType() == Message.MessageType.SUBGRAPH) { Iterable<BytesWritable> subgraphList = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Integer partitionID = Ints.fromByteArray(subgraphList.iterator().next().getBytes()); for (BytesWritable subgraphListElement : Iterables.skip(subgraphList, 1)) { LongWritable subgraphID = new LongWritable(Longs.fromByteArray(subgraphListElement.getBytes())); subgraphPartitionMap.put((K) subgraphID, partitionID); } continue; } /* * receiving query to find subgraph id Remote Vertex */ Iterable<BytesWritable> RemoteVertexQuery = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); /* * Reply format : sinkID1 subgraphID1 sinkID2 subgraphID2 ... */ Message<LongWritable, LongWritable> subgraphIDReply = new Message<LongWritable, LongWritable>(); controlInfo = new ControlMessage(); controlInfo.setTransmissionType(IControlMessage.TransmissionType.NORMAL); subgraphIDReply.setControlInfo(controlInfo); Integer sinkPartition = Ints.fromByteArray(RemoteVertexQuery.iterator().next().getBytes()); boolean hasAVertex = false; for (BytesWritable remoteVertex : Iterables.skip(RemoteVertexQuery, 1)) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(remoteVertex.getBytes())); LongWritable sinkSubgraphID = vertexSubgraphMap.get(sinkID); // In case this partition does not have the vertex /* * Case 1 : If vertex does not exist Case 2 : If vertex exists but is * remote, then its subgraphID is null */ if (sinkSubgraphID == null) { continue; } hasAVertex = true; byte sinkIDbytes[] = Longs.toByteArray(sinkID.get()); controlInfo.addextraInfo(sinkIDbytes); byte subgraphIDbytes[] = Longs.toByteArray(sinkSubgraphID.get()); controlInfo.addextraInfo(subgraphIDbytes); } if (hasAVertex) { peer.send(peer.getPeerName(sinkPartition.intValue()), (Message<K, M>) subgraphIDReply); } } peer.sync(); while ((msg = (Message<LongWritable, LongWritable>) peer.getCurrentMessage()) != null) { Iterable<BytesWritable> remoteVertexReply = ((ControlMessage) msg.getControlInfo()).getExtraInfo(); Iterator<BytesWritable> queryResponse = remoteVertexReply.iterator(); while (queryResponse.hasNext()) { LongWritable sinkID = new LongWritable(Longs.fromByteArray(queryResponse.next().getBytes())); LongWritable remoteSubgraphID = new LongWritable( Longs.fromByteArray(queryResponse.next().getBytes())); RemoteVertex<V, E, LongWritable, LongWritable, LongWritable> sink = (RemoteVertex<V, E, LongWritable, LongWritable, LongWritable>) vertexMap .get(sinkID); if (sink == null) { System.out.println("NULLLL" + sink); } sink.setSubgraphID(remoteSubgraphID); } } return partition.getSubgraphs(); }
From source file:io.amient.kafka.hadoop.testutils.MyJsonTimestampExtractor.java
License:Apache License
@Override public Long extract(MsgMetadataWritable key, BytesWritable value) throws IOException { if (value.getLength() > 0) { JsonNode json = jsonMapper.readValue(value.getBytes(), 0, value.getLength(), JsonNode.class); if (json.has("timestamp")) { return json.get("timestamp").getLongValue(); }/*from ww w.ja v a 2s.c o m*/ } return null; }
From source file:io.amient.kafka.hadoop.testutils.MyTextTimestampExtractor.java
License:Apache License
@Override public Long extract(MsgMetadataWritable key, BytesWritable value) throws IOException { try {/*from ww w. j av a 2 s . com*/ String leadString = new String(Arrays.copyOfRange(value.getBytes(), 0, 19)); return parser.parse(leadString).getTime(); } catch (ParseException e) { return null; } }
From source file:io.aos.hdfs.BytesWritableTest.java
License:Apache License
@Test public void test() throws IOException { // vv BytesWritableTest BytesWritable b = new BytesWritable(new byte[] { 3, 5 }); byte[] bytes = serialize(b); assertThat(StringUtils.byteToHexString(bytes), is("000000020305")); // ^^ BytesWritableTest // vv BytesWritableTest-Capacity b.setCapacity(11);/*from w ww .ja v a 2s . c o m*/ assertThat(b.getLength(), is(2)); assertThat(b.getBytes().length, is(11)); // ^^ BytesWritableTest-Capacity }
From source file:io.apigee.lembos.mapreduce.converters.input.BytesWritableConverter.java
License:Apache License
/** * Takes in a {@link BytesWritable} and returns a {@link byte[]}. * * @param scope the JavaScript scope//from w w w . ja v a2s. co m * @param writable the value to convert * * @return the byte[] equivalent */ @Override public Object toJavaScript(final Scriptable scope, final BytesWritable writable) { return writable.getBytes(); }