List of usage examples for org.apache.hadoop.net NetUtils isLocalAddress
public static boolean isLocalAddress(InetAddress addr)
From source file:com.datatorrent.stram.engine.StreamingContainer.java
License:Apache License
private HashMap.SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher( String connIdentifier, StreamCodec<?> streamCodec, long finishedWindowId, int queueCapacity, OperatorDeployInfo.OutputDeployInfo nodi) throws UnknownHostException { String sinkIdentifier = "tcp://".concat(nodi.bufferServerHost).concat(":") .concat(String.valueOf(nodi.bufferServerPort)).concat("/").concat(connIdentifier); StreamContext bssc = new StreamContext(nodi.declaredStreamId); bssc.setPortId(nodi.portName);/*from w w w. ja v a 2 s .co m*/ bssc.setSourceId(connIdentifier); bssc.setSinkId(sinkIdentifier); bssc.setFinishedWindowId(finishedWindowId); bssc.put(StreamContext.CODEC, streamCodec); bssc.put(StreamContext.EVENT_LOOP, eventloop); bssc.setBufferServerAddress( InetSocketAddress.createUnresolved(nodi.bufferServerHost, nodi.bufferServerPort)); bssc.put(StreamContext.BUFFER_SERVER_TOKEN, nodi.bufferServerToken); if (NetUtils.isLocalAddress(bssc.getBufferServerAddress().getAddress())) { bssc.setBufferServerAddress(new InetSocketAddress(InetAddress.getByName(null), nodi.bufferServerPort)); } Stream publisher = fastPublisherSubscriber ? new FastPublisher(connIdentifier, queueCapacity * 256) : new BufferServerPublisher(connIdentifier, queueCapacity); return new HashMap.SimpleEntry<String, ComponentContextPair<Stream, StreamContext>>(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(publisher, bssc)); }
From source file:com.datatorrent.stram.engine.StreamingContainer.java
License:Apache License
@SuppressWarnings("unchecked") private void deployInputStreams(List<OperatorDeployInfo> operatorList, HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams) throws UnknownHostException { /*//from www . j a va2s . co m * collect any input operators along with their smallest window id, * those are subsequently used to setup the window generator */ ArrayList<OperatorDeployInfo> inputNodes = new ArrayList<OperatorDeployInfo>(); long smallestCheckpointedWindowId = Long.MAX_VALUE; //a simple map which maps the oio node to it's the node which owns the thread. Map<Integer, Integer> oioNodes = new ConcurrentHashMap<Integer, Integer>(); /* * Hook up all the downstream ports. There are 2 places where we deal with more than 1 * downstream ports. The first one follows immediately for WindowGenerator. The second * case is when source for the input port of some node in this container is in another * container. So we need to create the stream. We need to track this stream along with * other streams,and many such streams may exist, we hash them against buffer server * info as we did for outputs but throw in the sinkid in the mix as well. */ for (OperatorDeployInfo ndi : operatorList) { if (ndi.inputs == null || ndi.inputs.isEmpty()) { /* * This has to be InputNode, so let's hook the WindowGenerator to it. * A node which does not take any input cannot exist in the DAG since it would be completely * unaware of the windows. So for that reason, AbstractInputNode allows Component.INPUT port. */ inputNodes.add(ndi); /* * When we activate the window Generator, we plan to activate it only from required windowId. */ ndi.checkpoint = getFinishedCheckpoint(ndi); if (ndi.checkpoint.windowId < smallestCheckpointedWindowId) { smallestCheckpointedWindowId = ndi.checkpoint.windowId; } } else { Node<?> node = nodes.get(ndi.id); for (OperatorDeployInfo.InputDeployInfo nidi : ndi.inputs) { if (nidi.streamCodecs.size() != 1) { throw new IllegalStateException("Only one input codec configuration should be present"); } Map.Entry<Integer, StreamCodec<?>> entry = nidi.streamCodecs.entrySet().iterator().next(); Integer streamCodecIdentifier = entry.getKey(); StreamCodec<?> streamCodec = entry.getValue(); String sourceIdentifier = Integer.toString(nidi.sourceNodeId).concat(Component.CONCAT_SEPARATOR) .concat(nidi.sourcePortName); String sinkIdentifier = Integer.toString(ndi.id).concat(Component.CONCAT_SEPARATOR) .concat(nidi.portName); int queueCapacity = getValue(PortContext.QUEUE_CAPACITY, nidi, ndi); Checkpoint checkpoint = getFinishedCheckpoint(ndi); ComponentContextPair<Stream, StreamContext> pair = streams.get(sourceIdentifier); if (pair == null) { pair = newStreams.get(sourceIdentifier); } if (pair == null) { /* * We connect to the buffer server for the input on this port. * We have already placed all the output streams for all the operators in this container. * Yet, there is no stream which can source this port so it has to come from the buffer * server, so let's make a connection to it. */ assert (nidi.locality != Locality.CONTAINER_LOCAL && nidi.locality != Locality.THREAD_LOCAL); StreamContext context = new StreamContext(nidi.declaredStreamId); context.setBufferServerAddress( InetSocketAddress.createUnresolved(nidi.bufferServerHost, nidi.bufferServerPort)); if (NetUtils.isLocalAddress(context.getBufferServerAddress().getAddress())) { context.setBufferServerAddress( new InetSocketAddress(InetAddress.getByName(null), nidi.bufferServerPort)); } context.put(StreamContext.BUFFER_SERVER_TOKEN, nidi.bufferServerToken); String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier; context.setPortId(nidi.portName); context.put(StreamContext.CODEC, streamCodec); context.put(StreamContext.EVENT_LOOP, eventloop); context.setPartitions(nidi.partitionMask, nidi.partitionKeys); //context.setSourceId(sourceIdentifier); context.setSourceId(connIdentifier); context.setSinkId(sinkIdentifier); context.setFinishedWindowId(checkpoint.windowId); BufferServerSubscriber subscriber = fastPublisherSubscriber ? new FastSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":") .concat(String.valueOf(nidi.bufferServerPort)).concat("/") .concat(connIdentifier), queueCapacity) : new BufferServerSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":") .concat(String.valueOf(nidi.bufferServerPort)).concat("/") .concat(connIdentifier), queueCapacity); SweepableReservoir reservoir = subscriber.acquireReservoir(sinkIdentifier, queueCapacity); if (checkpoint.windowId >= 0) { node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, reservoir, checkpoint.windowId)); } node.connectInputPort(nidi.portName, reservoir); newStreams.put(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(subscriber, context)); logger.debug("put input stream {} against key {}", subscriber, sinkIdentifier); } else { assert (nidi.locality == Locality.CONTAINER_LOCAL || nidi.locality == Locality.THREAD_LOCAL); /* we are still dealing with the MuxStream originating at the output of the source port */ StreamContext inlineContext = new StreamContext(nidi.declaredStreamId); inlineContext.setSourceId(sourceIdentifier); inlineContext.setSinkId(sinkIdentifier); Stream stream; switch (nidi.locality) { case CONTAINER_LOCAL: int outputQueueCapacity = getOutputQueueCapacity(operatorList, nidi.sourceNodeId, nidi.sourcePortName); if (outputQueueCapacity > queueCapacity) { queueCapacity = outputQueueCapacity; } stream = new InlineStream(queueCapacity); if (checkpoint.windowId >= 0) { node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, (SweepableReservoir) stream, checkpoint.windowId)); } break; case THREAD_LOCAL: stream = new OiOStream(); oioNodes.put(ndi.id, nidi.sourceNodeId); break; default: throw new IllegalStateException("Locality can be either ContainerLocal or ThreadLocal"); } node.connectInputPort(nidi.portName, (SweepableReservoir) stream); newStreams.put(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(stream, inlineContext)); if (!(pair.component instanceof Stream.MultiSinkCapableStream)) { String originalSinkId = pair.context.getSinkId(); /* we come here only if we are trying to augment the dag */ StreamContext muxContext = new StreamContext(nidi.declaredStreamId); muxContext.setSourceId(sourceIdentifier); muxContext.setFinishedWindowId(checkpoint.windowId); muxContext.setSinkId(originalSinkId); MuxStream muxStream = new MuxStream(); muxStream.setSink(originalSinkId, pair.component); streams.put(originalSinkId, pair); Node<?> sourceNode = nodes.get(nidi.sourceNodeId); sourceNode.connectOutputPort(nidi.sourcePortName, muxStream); newStreams.put(sourceIdentifier, pair = new ComponentContextPair<Stream, StreamContext>(muxStream, muxContext)); } /* here everything should be multisink capable */ if (nidi.partitionKeys == null || nidi.partitionKeys.isEmpty()) { ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, stream); } else { /* * generally speaking we do not have partitions on the inline streams so the control should not * come here but if it comes, then we are ready to handle it using the partition aware streams. */ PartitionAwareSink<Object> pas = new PartitionAwareSink<Object>( streamCodec == null ? nonSerializingStreamCodec : (StreamCodec<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream); ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas); } String streamSinkId = pair.context.getSinkId(); if (streamSinkId == null) { pair.context.setSinkId(sinkIdentifier); } else { pair.context.setSinkId(streamSinkId.concat(", ").concat(sinkIdentifier)); } } } } } setupOiOGroups(oioNodes); if (!inputNodes.isEmpty()) { WindowGenerator windowGenerator = setupWindowGenerator(smallestCheckpointedWindowId); for (OperatorDeployInfo ndi : inputNodes) { generators.put(ndi.id, windowGenerator); Node<?> node = nodes.get(ndi.id); SweepableReservoir reservoir = windowGenerator.acquireReservoir(String.valueOf(ndi.id), 1024); if (ndi.checkpoint.windowId >= 0) { node.connectInputPort(Node.INPUT, new WindowIdActivatedReservoir(Integer.toString(ndi.id), reservoir, ndi.checkpoint.windowId)); } node.connectInputPort(Node.INPUT, reservoir); } } }
From source file:com.mellanox.r4h.DFSClient.java
License:Apache License
public static boolean isLocalAddress(InetSocketAddress targetAddr) { InetAddress addr = targetAddr.getAddress(); Boolean cached = localAddrMap.get(addr.getHostAddress()); if (cached != null) { if (LOG.isTraceEnabled()) { LOG.trace("Address " + targetAddr + (cached ? " is local" : " is not local")); }//from w w w.j av a 2 s. c o m return cached; } boolean local = NetUtils.isLocalAddress(addr); if (LOG.isTraceEnabled()) { LOG.trace("Address " + targetAddr + (local ? " is local" : " is not local")); } localAddrMap.put(addr.getHostAddress(), local); return local; }
From source file:org.apache.atlas.ha.AtlasServerIdSelector.java
License:Apache License
/** * Return the ID corresponding to this Atlas instance. * * The match is done by looking for an ID configured in {@link HAConfiguration#ATLAS_SERVER_IDS} key * that has a host:port entry for the key {@link HAConfiguration#ATLAS_SERVER_ADDRESS_PREFIX}+ID where * the host is a local IP address and port is set in the system property * {@link AtlasConstants#SYSTEM_PROPERTY_APP_PORT}. * * @param configuration// w w w .j av a 2s.c o m * @return * @throws AtlasException if no ID is found that maps to a local IP Address or port */ public static String selectServerId(Configuration configuration) throws AtlasException { // ids are already trimmed by this method String[] ids = configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS); String matchingServerId = null; int appPort = Integer.parseInt(System.getProperty(AtlasConstants.SYSTEM_PROPERTY_APP_PORT)); for (String id : ids) { String hostPort = configuration.getString(HAConfiguration.ATLAS_SERVER_ADDRESS_PREFIX + id); if (!StringUtils.isEmpty(hostPort)) { InetSocketAddress socketAddress; try { socketAddress = NetUtils.createSocketAddr(hostPort); } catch (Exception e) { LOG.warn("Exception while trying to get socket address for " + hostPort, e); continue; } if (!socketAddress.isUnresolved() && NetUtils.isLocalAddress(socketAddress.getAddress()) && appPort == socketAddress.getPort()) { LOG.info("Found matched server id " + id + " with host port: " + hostPort); matchingServerId = id; break; } } else { LOG.info("Could not find matching address entry for id: " + id); } } if (matchingServerId == null) { String msg = String.format( "Could not find server id for this instance. " + "Unable to find IDs matching any local host and port binding among %s", StringUtils.join(ids, ",")); throw new AtlasException(msg); } return matchingServerId; }