List of usage examples for com.google.common.graph MutableNetwork removeNode
@CanIgnoreReturnValue
boolean removeNode(Object node);
From source file:org.apache.beam.runners.dataflow.worker.graph.Networks.java
/** * Applies the {@code function} to all nodes within the {@code network}. Replaces any node which * is not {@link #equals(Object)} to the original node, maintaining all existing edges between * nodes./*from w w w. j a va 2 s.c o m*/ */ public static <N, E> void replaceDirectedNetworkNodes(MutableNetwork<N, E> network, Function<N, N> function) { checkArgument(network.isDirected(), "Only directed networks are supported, given %s", network); checkArgument(!network.allowsSelfLoops(), "Only networks without self loops are supported, given %s", network); // A map from the existing node to the replacement node Map<N, N> oldNodesToNewNodes = new HashMap<>(network.nodes().size()); for (N currentNode : network.nodes()) { N newNode = function.apply(currentNode); // Skip updating the network if the old node is equivalent to the new node if (!currentNode.equals(newNode)) { oldNodesToNewNodes.put(currentNode, newNode); } } // For each replacement, connect up the existing predecessors and successors to the new node // and then remove the old node. for (Map.Entry<N, N> entry : oldNodesToNewNodes.entrySet()) { N oldNode = entry.getKey(); N newNode = entry.getValue(); network.addNode(newNode); for (N predecessor : ImmutableSet.copyOf(network.predecessors(oldNode))) { for (E edge : ImmutableSet.copyOf(network.edgesConnecting(predecessor, oldNode))) { network.removeEdge(edge); network.addEdge(predecessor, newNode, edge); } } for (N successor : ImmutableSet.copyOf(network.successors(oldNode))) { for (E edge : ImmutableSet.copyOf(network.edgesConnecting(oldNode, successor))) { network.removeEdge(edge); network.addEdge(newNode, successor, edge); } } network.removeNode(oldNode); } }
From source file:org.apache.beam.runners.dataflow.worker.graph.RemoveFlattenInstructionsFunction.java
@Override public MutableNetwork<Node, Edge> apply(MutableNetwork<Node, Edge> network) { for (Node node : ImmutableList.copyOf(Iterables.filter(network.nodes(), IsFlatten.INSTANCE))) { // For each successor instruction after the Flatten, connect it directly to the // predecessor PCollections of Flatten. Node flattenPCollection = Iterables.getOnlyElement(network.successors(node)); for (Node successorInstruction : ImmutableList.copyOf(network.successors(flattenPCollection))) { for (Edge edge : ImmutableList .copyOf(network.edgesConnecting(flattenPCollection, successorInstruction))) { for (Node predecessorPCollection : ImmutableList.copyOf(network.predecessors(node))) { network.addEdge(predecessorPCollection, successorInstruction, edge.clone()); }/* w w w . jav a 2 s . c o m*/ } } // Remove the Flatten instruction and its output PCollection. network.removeNode(flattenPCollection); network.removeNode(node); } return network; }
From source file:org.apache.beam.runners.core.construction.graph.Networks.java
/** * Applies the {@code function} to all nodes within the {@code network}. Replaces any node which * is not {@link #equals(Object)} to the original node, maintaining all existing edges between * nodes.//w w w . jav a 2s. com */ public static <NodeT, EdgeT> void replaceDirectedNetworkNodes(MutableNetwork<NodeT, EdgeT> network, Function<NodeT, NodeT> function) { checkArgument(network.isDirected(), "Only directed networks are supported, given %s", network); checkArgument(!network.allowsSelfLoops(), "Only networks without self loops are supported, given %s", network); // A map from the existing node to the replacement node Map<NodeT, NodeT> oldNodesToNewNodes = new HashMap<>(network.nodes().size()); for (NodeT currentNode : network.nodes()) { NodeT newNode = function.apply(currentNode); // Skip updating the network if the old node is equivalent to the new node if (!currentNode.equals(newNode)) { oldNodesToNewNodes.put(currentNode, newNode); } } // For each replacement, connect up the existing predecessors and successors to the new node // and then remove the old node. for (Map.Entry<NodeT, NodeT> entry : oldNodesToNewNodes.entrySet()) { NodeT oldNode = entry.getKey(); NodeT newNode = entry.getValue(); network.addNode(newNode); for (NodeT predecessor : ImmutableSet.copyOf(network.predecessors(oldNode))) { for (EdgeT edge : ImmutableSet.copyOf(network.edgesConnecting(predecessor, oldNode))) { network.removeEdge(edge); network.addEdge(predecessor, newNode, edge); } } for (NodeT successor : ImmutableSet.copyOf(network.successors(oldNode))) { for (EdgeT edge : ImmutableSet.copyOf(network.edgesConnecting(oldNode, successor))) { network.removeEdge(edge); network.addEdge(newNode, successor, edge); } } network.removeNode(oldNode); } }
From source file:org.apache.beam.runners.core.construction.graph.Networks.java
/** * Compute the topological order for a {@link Network}. * * <p>Nodes must be considered in the order specified by the {@link Network Network's} {@link * Network#nodeOrder()}. This ensures that any two Networks with the same nodes and node orders * produce the same result./* www . j a v a2 s. c o m*/ */ private static <NodeT> Iterable<NodeT> computeTopologicalOrder(MutableNetwork<NodeT, ?> network) { // TODO: (github/guava/2641) Upgrade Guava and remove this method if topological sorting becomes // supported externally or remove this comment if its not going to be supported externally. checkArgument(network.isDirected(), "Only directed networks are supported, given %s", network); checkArgument(!network.allowsSelfLoops(), "Only networks without self loops are supported, given %s", network); // Uses the following algorithm: // A FAST & EFFECTIVE HEURISTIC FOR THE FEEDBACK ARC SET PROBLEM // Peter Eades, Xuemin Lin, W. F. Smyth // https://pdfs.semanticscholar.org/c7ed/d9acce96ca357876540e19664eb9d976637f.pdf // // The only edges that are ignored by the algorithm are back edges. // The algorithm (while there are still nodes in the graph): // 1) Removes all sinks from the graph adding them to the beginning of "s2". Continue to do this till there // are no more sinks. // 2) Removes all source from the graph adding them to the end of "s1". Continue to do this till there // are no more sources. // 3) Remote a single node with the highest delta within the graph and add it to the end of "s1". // // The topological order is then the s1 concatenated with s2. Deque<NodeT> s1 = new ArrayDeque<>(); Deque<NodeT> s2 = new ArrayDeque<>(); Ordering<NodeT> maximumOrdering = new Ordering<NodeT>() { @Override public int compare(NodeT t0, NodeT t1) { return (network.outDegree(t0) - network.inDegree(t0)) - (network.outDegree(t1) - network.inDegree(t1)); } }; while (!network.nodes().isEmpty()) { boolean nodeRemoved; do { nodeRemoved = false; for (NodeT possibleSink : ImmutableList.copyOf(network.nodes())) { if (network.outDegree(possibleSink) == 0) { network.removeNode(possibleSink); s2.addFirst(possibleSink); nodeRemoved = true; } } } while (nodeRemoved); do { nodeRemoved = false; for (NodeT possibleSource : ImmutableList.copyOf(network.nodes())) { if (network.inDegree(possibleSource) == 0) { network.removeNode(possibleSource); s1.addLast(possibleSource); nodeRemoved = true; } } } while (nodeRemoved); if (!network.nodes().isEmpty()) { NodeT maximum = maximumOrdering.max(network.nodes()); network.removeNode(maximum); s1.addLast(maximum); } } return ImmutableList.<NodeT>builder().addAll(s1).addAll(s2).build(); }
From source file:org.apache.beam.runners.dataflow.worker.graph.CloneAmbiguousFlattensFunction.java
/** * A helper function which performs the actual cloning procedure, which means creating the runner * and SDK versions of both the ambiguous flatten and its PCollection, attaching the old flatten's * predecessors and successors properly, and then removing the ambiguous flatten from the network. *//*from w w w. j a v a2 s .co m*/ private void cloneFlatten(Node flatten, MutableNetwork<Node, Edge> network) { // Start by creating the clones of the flatten and its PCollection. InstructionOutputNode flattenOut = (InstructionOutputNode) Iterables .getOnlyElement(network.successors(flatten)); ParallelInstruction flattenInstruction = ((ParallelInstructionNode) flatten).getParallelInstruction(); Node runnerFlatten = ParallelInstructionNode.create(flattenInstruction, ExecutionLocation.RUNNER_HARNESS); Node runnerFlattenOut = InstructionOutputNode.create(flattenOut.getInstructionOutput(), flattenOut.getPcollectionId()); network.addNode(runnerFlatten); network.addNode(runnerFlattenOut); Node sdkFlatten = ParallelInstructionNode.create(flattenInstruction, ExecutionLocation.SDK_HARNESS); Node sdkFlattenOut = InstructionOutputNode.create(flattenOut.getInstructionOutput(), flattenOut.getPcollectionId()); network.addNode(sdkFlatten); network.addNode(sdkFlattenOut); for (Edge edge : ImmutableList.copyOf(network.edgesConnecting(flatten, flattenOut))) { network.addEdge(runnerFlatten, runnerFlattenOut, edge.clone()); network.addEdge(sdkFlatten, sdkFlattenOut, edge.clone()); } // Copy over predecessor edges to both cloned nodes. for (Node predecessor : network.predecessors(flatten)) { for (Edge edge : ImmutableList.copyOf(network.edgesConnecting(predecessor, flatten))) { network.addEdge(predecessor, runnerFlatten, edge.clone()); network.addEdge(predecessor, sdkFlatten, edge.clone()); } } // Copy over successor edges depending on execution locations of successors. for (Node successor : network.successors(flattenOut)) { // Connect successor to SDK harness only if sure it executes in SDK. Node selectedOutput = executesInSdkHarness(successor) ? sdkFlattenOut : runnerFlattenOut; for (Edge edge : ImmutableList.copyOf(network.edgesConnecting(flattenOut, successor))) { network.addEdge(selectedOutput, successor, edge.clone()); } } network.removeNode(flatten); network.removeNode(flattenOut); }
From source file:org.apache.beam.runners.dataflow.worker.graph.CreateRegisterFnOperationFunction.java
@Override public MutableNetwork<Node, Edge> apply(MutableNetwork<Node, Edge> network) { // Record all SDK nodes, and all root nodes. Set<Node> runnerRootNodes = new HashSet<>(); Set<Node> sdkNodes = new HashSet<>(); Set<Node> sdkRootNodes = new HashSet<>(); for (ParallelInstructionNode node : Iterables.filter(network.nodes(), ParallelInstructionNode.class)) { if (executesInSdkHarness(node)) { sdkNodes.add(node);//from w w w . ja va 2 s. co m if (network.inDegree(node) == 0) { sdkRootNodes.add(node); } } else if (network.inDegree(node) == 0) { runnerRootNodes.add(node); } } // If nothing executes within the SDK harness, return the original network. if (sdkNodes.isEmpty()) { return network; } // Represents the set of nodes which represent gRPC boundaries from the Runner to the SDK. Set<Node> runnerToSdkBoundaries = new HashSet<>(); // Represents the set of nodes which represent gRPC boundaries from the SDK to the Runner. Set<Node> sdkToRunnerBoundaries = new HashSet<>(); ImmutableNetwork<Node, Edge> originalNetwork = ImmutableNetwork.copyOf(network); // Update the network with outputs which are meant to bridge the instructions // that execute in different harnesses. One output per direction of information // flow from runner to SDK and SDK to runner per original output node. for (InstructionOutputNode outputNode : Iterables.filter(originalNetwork.nodes(), InstructionOutputNode.class)) { // Categorize all predecessor instructions Set<Node> predecessorRunnerInstructions = new HashSet<>(); Set<Node> predecessorSdkInstructions = new HashSet<>(); for (Node predecessorInstruction : originalNetwork.predecessors(outputNode)) { if (sdkNodes.contains(predecessorInstruction)) { predecessorSdkInstructions.add(predecessorInstruction); } else { predecessorRunnerInstructions.add(predecessorInstruction); } } // Categorize all successor instructions Set<Node> successorRunnerInstructions = new HashSet<>(); Set<Node> successorSdkInstructions = new HashSet<>(); for (Node successorInstruction : originalNetwork.successors(outputNode)) { if (sdkNodes.contains(successorInstruction)) { successorSdkInstructions.add(successorInstruction); } else { successorRunnerInstructions.add(successorInstruction); } } // If there is data that will be flowing from the Runner to the SDK, rewire network to have // nodes connected across a gRPC node. Also add the gRPC node as an SDK root. if (!predecessorRunnerInstructions.isEmpty() && !successorSdkInstructions.isEmpty()) { runnerToSdkBoundaries.add(rewireAcrossSdkRunnerPortNode(network, outputNode, predecessorRunnerInstructions, successorSdkInstructions)); } // If there is data that will be flowing from the SDK to the Runner, rewire network to have // nodes connected across a gRPC node. if (!predecessorSdkInstructions.isEmpty() && !successorRunnerInstructions.isEmpty()) { sdkToRunnerBoundaries.add(rewireAcrossSdkRunnerPortNode(network, outputNode, predecessorSdkInstructions, successorRunnerInstructions)); } // Remove original output node if it was rewired because it will have become disconnected // through the new output node. if (network.inDegree(outputNode) == 0) { network.removeNode(outputNode); } } // Create the subnetworks that represent potentially multiple fused SDK portions and a single // fused Runner portion replacing the SDK portion that is embedded within the Runner portion // with a RegisterFnOperation, adding edges to maintain proper happens before relationships. Set<Node> allRunnerNodes = Networks.reachableNodes(network, Sets.union(runnerRootNodes, sdkToRunnerBoundaries), runnerToSdkBoundaries); if (this.useExecutableStageBundleExecution) { // When using shared library, there is no grpc node in runner graph. allRunnerNodes = Sets.difference(allRunnerNodes, Sets.union(runnerToSdkBoundaries, sdkToRunnerBoundaries)); } MutableNetwork<Node, Edge> runnerNetwork = Graphs.inducedSubgraph(network, allRunnerNodes); // TODO: Reduce the amount of 'copying' of SDK nodes by breaking potential cycles // between the SDK networks and the Runner network. Cycles can occur because entire // SDK subnetworks are replaced by a singular node within the Runner network. // khines@ suggested to look at go/priority-based-fusion for an algorithm based upon // using poison paths. for (Node sdkRoot : Sets.union(sdkRootNodes, runnerToSdkBoundaries)) { Set<Node> sdkSubnetworkNodes = Networks.reachableNodes(network, ImmutableSet.of(sdkRoot), sdkToRunnerBoundaries); MutableNetwork<Node, Edge> sdkNetwork = Graphs.inducedSubgraph(network, sdkSubnetworkNodes); Node registerFnNode = registerFnOperationFunction.apply(sdkNetwork); runnerNetwork.addNode(registerFnNode); // Create happens before relationships between all Runner and SDK nodes which are in the // SDK subnetwork; direction dependent on whether its a predecessor of the SDK subnetwork or // a successor. if (this.useExecutableStageBundleExecution) { // When using shared library, there is no gprc node in runner graph. Then the registerFnNode // should be linked directly to 2 OutputInstruction nodes. for (Node predecessor : Sets.intersection(sdkSubnetworkNodes, runnerToSdkBoundaries)) { predecessor = network.predecessors(predecessor).iterator().next(); runnerNetwork.addEdge(predecessor, registerFnNode, HappensBeforeEdge.create()); } for (Node successor : Sets.intersection(sdkSubnetworkNodes, sdkToRunnerBoundaries)) { successor = network.successors(successor).iterator().next(); runnerNetwork.addEdge(registerFnNode, successor, HappensBeforeEdge.create()); } } else { for (Node predecessor : Sets.intersection(sdkSubnetworkNodes, runnerToSdkBoundaries)) { runnerNetwork.addEdge(predecessor, registerFnNode, HappensBeforeEdge.create()); } for (Node successor : Sets.intersection(sdkSubnetworkNodes, sdkToRunnerBoundaries)) { runnerNetwork.addEdge(registerFnNode, successor, HappensBeforeEdge.create()); } } } return runnerNetwork; }