List of usage examples for com.google.common.graph Network successors
@Override Set<N> successors(Object node);
From source file:org.apache.beam.runners.dataflow.worker.IntrinsicMapTaskExecutorFactory.java
static OutputReceiver[] getOutputReceivers(Network<Node, Edge> network, Node node) { int outDegree = network.outDegree(node); if (outDegree == 0) { return EMPTY_OUTPUT_RECEIVER_ARRAY; }/*from ww w . j a v a 2 s . com*/ OutputReceiver[] receivers = new OutputReceiver[outDegree]; Iterator<Node> receiverNodes = network.successors(node).iterator(); int i = 0; do { receivers[i] = ((OutputReceiverNode) receiverNodes.next()).getOutputReceiver(); i += 1; } while (receiverNodes.hasNext()); return receivers; }
From source file:org.apache.beam.runners.dataflow.worker.graph.Networks.java
/** * Returns the set of nodes that are reachable from {@code startNodes} up to and including {@code * endNodes}. Node B is defined as reachable from node A if there exists a path (a sequence of * adjacent outgoing edges) starting at node A and ending at node B which does not pass through * any node in {@code endNodes}. Note that a node is always reachable from itself via a * zero-length path./*from www . j av a 2 s .c o m*/ * * <p>This is a "snapshot" based on the current topology of the {@code network}, rather than a * live view of the set of nodes reachable from {@code node}. In other words, the returned {@link * Set} will not be updated after modifications to the {@code network}. */ public static <N, E> Set<N> reachableNodes(Network<N, E> network, Set<N> startNodes, Set<N> endNodes) { Set<N> visitedNodes = new HashSet<>(); Queue<N> queuedNodes = new ArrayDeque<>(); queuedNodes.addAll(startNodes); // Perform a breadth-first traversal rooted at the input node. while (!queuedNodes.isEmpty()) { N currentNode = queuedNodes.remove(); // If we have already visited this node or it is a terminal node than do not add any // successors. if (!visitedNodes.add(currentNode) || endNodes.contains(currentNode)) { continue; } queuedNodes.addAll(network.successors(currentNode)); } return visitedNodes; }
From source file:org.apache.beam.runners.core.construction.graph.Networks.java
/** * Returns the set of nodes that are reachable from {@code startNodes} up to and including {@code * endNodes}. Node B is defined as reachable from node A if there exists a path (a sequence of * adjacent outgoing edges) starting at node A and ending at node B which does not pass through * any node in {@code endNodes}. Note that a node is always reachable from itself via a * zero-length path./*from w ww. ja v a 2 s . co m*/ * * <p>This is a "snapshot" based on the current topology of the {@code network}, rather than a * live view of the set of nodes reachable from {@code node}. In other words, the returned {@link * Set} will not be updated after modifications to the {@code network}. */ public static <NodeT, EdgeT> Set<NodeT> reachableNodes(Network<NodeT, EdgeT> network, Set<NodeT> startNodes, Set<NodeT> endNodes) { Set<NodeT> visitedNodes = new HashSet<>(); Queue<NodeT> queuedNodes = new ArrayDeque<>(); queuedNodes.addAll(startNodes); // Perform a breadth-first traversal rooted at the input node. while (!queuedNodes.isEmpty()) { NodeT currentNode = queuedNodes.remove(); // If we have already visited this node or it is a terminal node than do not add any // successors. if (!visitedNodes.add(currentNode) || endNodes.contains(currentNode)) { continue; } queuedNodes.addAll(network.successors(currentNode)); } return visitedNodes; }
From source file:org.apache.beam.runners.dataflow.worker.graph.Networks.java
/** Returns a set of nodes sorted in topological order. */ public static <N, E> Set<N> topologicalOrder(Network<N, E> network) { // TODO: Upgrade Guava and remove this method if topological sorting becomes // supported externally or remove this comment if its not going to be supported externally. checkArgument(network.isDirected(), "Only directed networks are supported, given %s", network); checkArgument(!network.allowsSelfLoops(), "Only networks without self loops are supported, given %s", network);//from w w w .j av a 2 s. c o m // Linked hashset will prevent duplicates from appearing and will maintain insertion order. LinkedHashSet<N> nodes = new LinkedHashSet<>(network.nodes().size()); Queue<N> processingOrder = new ArrayDeque<>(); // Add all the roots for (N node : network.nodes()) { if (network.inDegree(node) == 0) { processingOrder.add(node); } } while (!processingOrder.isEmpty()) { N current = processingOrder.remove(); // If all predecessors have already been added, then we can add this node, otherwise // we need to add the node to the back of the processing queue. if (nodes.containsAll(network.predecessors(current))) { nodes.add(current); processingOrder.addAll(network.successors(current)); } else { processingOrder.add(current); } } return nodes; }
From source file:org.apache.beam.runners.dataflow.worker.IntrinsicMapTaskExecutorFactory.java
private OperationNode createParDoOperation(Network<Node, Edge> network, ParallelInstructionNode node, PipelineOptions options, DataflowExecutionContext<?> executionContext, DataflowOperationContext operationContext) throws Exception { ParallelInstruction instruction = node.getParallelInstruction(); ParDoInstruction parDo = instruction.getParDo(); TupleTag<?> mainOutputTag = tupleTag(parDo.getMultiOutputInfos().get(0)); ImmutableMap.Builder<TupleTag<?>, Integer> outputTagsToReceiverIndicesBuilder = ImmutableMap.builder(); int successorOffset = 0; for (Node successor : network.successors(node)) { for (Edge edge : network.edgesConnecting(node, successor)) { outputTagsToReceiverIndicesBuilder.put(tupleTag(((MultiOutputInfoEdge) edge).getMultiOutputInfo()), successorOffset);/* w w w . j a va2s. c om*/ } successorOffset += 1; } ParDoFn fn = parDoFnFactory.create(options, CloudObject.fromSpec(parDo.getUserFn()), parDo.getSideInputs(), mainOutputTag, outputTagsToReceiverIndicesBuilder.build(), executionContext, operationContext); OutputReceiver[] receivers = getOutputReceivers(network, node); return OperationNode.create(new ParDoOperation(fn, receivers, operationContext)); }
From source file:org.apache.beam.runners.dataflow.worker.BeamFnMapTaskExecutorFactory.java
private Function<Node, Node> createOperationTransformForExecutableStageNode(final Network<Node, Edge> network, final String stageName, final DataflowExecutionContext<?> executionContext, final JobBundleFactory jobBundleFactory) { return new TypeSafeNodeFunction<ExecutableStageNode>(ExecutableStageNode.class) { @Override/*from w w w . java 2 s .com*/ public Node typedApply(ExecutableStageNode input) { StageBundleFactory stageBundleFactory = jobBundleFactory.forStage(input.getExecutableStage()); Iterable<OutputReceiverNode> outputReceiverNodes = Iterables.filter(network.successors(input), OutputReceiverNode.class); Map<String, OutputReceiver> outputReceiverMap = new HashMap<>(); Lists.newArrayList(outputReceiverNodes).stream().forEach(outputReceiverNode -> outputReceiverMap .put(outputReceiverNode.getPcollectionId(), outputReceiverNode.getOutputReceiver())); return OperationNode.create(new ProcessRemoteBundleOperation( executionContext.createOperationContext( NameContext.create(stageName, stageName, stageName, stageName)), stageBundleFactory, outputReceiverMap)); } }; }
From source file:org.apache.beam.runners.dataflow.worker.BeamFnMapTaskExecutorFactory.java
private Function<Node, Node> createOperationTransformForGrpcPortNodes(final Network<Node, Edge> network, final FnDataService beamFnDataService, final OperationContext context) { return new TypeSafeNodeFunction<RemoteGrpcPortNode>(RemoteGrpcPortNode.class) { @Override/*from www . j a va 2 s .c o m*/ public Node typedApply(RemoteGrpcPortNode input) { RegisterAndProcessBundleOperation registerFnOperation = (RegisterAndProcessBundleOperation) Iterables .getOnlyElement(Iterables.filter(network.adjacentNodes(input), OperationNode.class)) .getOperation(); // The coder comes from the one and only adjacent output node Coder<?> coder = Iterables .getOnlyElement(Iterables.filter(network.adjacentNodes(input), OutputReceiverNode.class)) .getCoder(); // We figure out whether we are outputting some where if the output node is a // successor. Iterable<OutputReceiverNode> outputReceiverNodes = Iterables.filter(network.successors(input), OutputReceiverNode.class); Operation operation; if (outputReceiverNodes.iterator().hasNext()) { Target target = Target.newBuilder() .setPrimitiveTransformReference(input.getPrimitiveTransformId()) .setName(input.getOutputId()).build(); OutputReceiver[] outputReceivers = new OutputReceiver[] { Iterables.getOnlyElement(outputReceiverNodes).getOutputReceiver() }; operation = new RemoteGrpcPortReadOperation<>(beamFnDataService, target, registerFnOperation::getProcessBundleInstructionId, (Coder) coder, outputReceivers, context); } else { Target target = Target.newBuilder() .setPrimitiveTransformReference(input.getPrimitiveTransformId()) .setName(input.getInputId()).build(); operation = new RemoteGrpcPortWriteOperation<>(beamFnDataService, target, registerFnOperation::getProcessBundleInstructionId, (Coder) coder, context); } return OperationNode.create(operation); } }; }