List of usage examples for java.net InetSocketAddress createUnresolved
public static InetSocketAddress createUnresolved(String host, int port)
From source file:com.datatorrent.stram.StreamingContainerManagerTest.java
public static StreamingContainerAgent assignContainer(StreamingContainerManager scm, String containerId) { return scm.assignContainer(new ContainerResource(0, containerId, "localhost", 1024, 0, null), InetSocketAddress.createUnresolved(containerId + "Host", 0)); }
From source file:com.datatorrent.stram.client.StramClientUtils.java
public static InetSocketAddress getRMWebAddress(Configuration conf, boolean sslEnabled, String rmId) { rmId = (rmId == null) ? "" : ("." + rmId); InetSocketAddress address;//from w w w .j av a 2 s . c om if (sslEnabled) { address = conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + rmId, YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT); } else { address = conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS + rmId, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); } LOG.info("rm webapp address setting {}", address); LOG.debug("rm setting sources {}", conf.getPropertySources(YarnConfiguration.RM_WEBAPP_ADDRESS)); InetSocketAddress resolvedSocketAddress = NetUtils.getConnectAddress(address); InetAddress resolved = resolvedSocketAddress.getAddress(); if (resolved == null || resolved.isAnyLocalAddress() || resolved.isLoopbackAddress()) { try { resolvedSocketAddress = InetSocketAddress .createUnresolved(InetAddress.getLocalHost().getCanonicalHostName(), address.getPort()); } catch (UnknownHostException e) { //Ignore and fallback. } } return resolvedSocketAddress; }
From source file:org.apache.hadoop.yarn.server.resourcemanager.security.TestDelegationTokenRenewer.java
@Test(timeout = 20000) public void testConcurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException { final CyclicBarrier startBarrier = new CyclicBarrier(2); final CyclicBarrier endBarrier = new CyclicBarrier(2); // this token uses barriers to block during renew final Credentials creds1 = new Credentials(); final Token<DelegationTokenIdentifier> token1 = mock(Token.class); when(token1.getKind()).thenReturn(KIND); DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"), new Text("user1")); when(token1.decodeIdentifier()).thenReturn(dtId1); creds1.addToken(new Text("token"), token1); doReturn(true).when(token1).isManaged(); doAnswer(new Answer<Long>() { public Long answer(InvocationOnMock invocation) throws InterruptedException, BrokenBarrierException { startBarrier.await();/*from w ww . j av a 2s . c o m*/ endBarrier.await(); return Long.MAX_VALUE; } }).when(token1).renew(any(Configuration.class)); // this dummy token fakes renewing final Credentials creds2 = new Credentials(); final Token<DelegationTokenIdentifier> token2 = mock(Token.class); when(token2.getKind()).thenReturn(KIND); when(token2.decodeIdentifier()).thenReturn(dtId1); creds2.addToken(new Text("token"), token2); doReturn(true).when(token2).isManaged(); doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class)); // fire up the renewer final DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter); RMContext mockContext = mock(RMContext.class); when(mockContext.getSystemCredentialsForApps()) .thenReturn(new ConcurrentHashMap<ApplicationId, ByteBuffer>()); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); dtr.setRMContext(mockContext); when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); dtr.init(conf); dtr.start(); // submit a job that blocks during renewal Thread submitThread = new Thread() { @Override public void run() { dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user"); } }; submitThread.start(); // wait till 1st submit blocks, then submit another startBarrier.await(); dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user"); // signal 1st to complete endBarrier.await(); submitThread.join(); }
From source file:com.datatorrent.stram.engine.StreamingContainer.java
private HashMap.SimpleEntry<String, ComponentContextPair<Stream, StreamContext>> deployBufferServerPublisher( String connIdentifier, StreamCodec<?> streamCodec, long finishedWindowId, int queueCapacity, OperatorDeployInfo.OutputDeployInfo nodi) throws UnknownHostException { String sinkIdentifier = "tcp://".concat(nodi.bufferServerHost).concat(":") .concat(String.valueOf(nodi.bufferServerPort)).concat("/").concat(connIdentifier); StreamContext bssc = new StreamContext(nodi.declaredStreamId); bssc.setPortId(nodi.portName);//from w w w. j av a 2 s.c o m bssc.setSourceId(connIdentifier); bssc.setSinkId(sinkIdentifier); bssc.setFinishedWindowId(finishedWindowId); bssc.put(StreamContext.CODEC, streamCodec); bssc.put(StreamContext.EVENT_LOOP, eventloop); bssc.setBufferServerAddress( InetSocketAddress.createUnresolved(nodi.bufferServerHost, nodi.bufferServerPort)); bssc.put(StreamContext.BUFFER_SERVER_TOKEN, nodi.bufferServerToken); InetAddress inetAddress = bssc.getBufferServerAddress().getAddress(); if (inetAddress != null && NetUtils.isLocalAddress(inetAddress)) { bssc.setBufferServerAddress(new InetSocketAddress(InetAddress.getByName(null), nodi.bufferServerPort)); } Stream publisher = fastPublisherSubscriber ? new FastPublisher(connIdentifier, queueCapacity * 256) : new BufferServerPublisher(connIdentifier, queueCapacity); return new HashMap.SimpleEntry<String, ComponentContextPair<Stream, StreamContext>>(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(publisher, bssc)); }
From source file:com.photon.phresco.framework.impl.SCMManagerImpl.java
void additionalAuthentication(String passPhrase) { final String passwordPhrase = passPhrase; JschConfigSessionFactory sessionFactory = new JschConfigSessionFactory() { @Override//from w w w . j av a2 s . c o m protected void configure(OpenSshConfig.Host hc, Session session) { CredentialsProvider provider = new CredentialsProvider() { @Override public boolean isInteractive() { return false; } @Override public boolean supports(CredentialItem... items) { return true; } @Override public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem { for (CredentialItem item : items) { if (item instanceof CredentialItem.StringType) { ((CredentialItem.StringType) item).setValue(passwordPhrase); } } return true; } }; UserInfo userInfo = new CredentialsProviderUserInfo(session, provider); // Unknown host key for ssh java.util.Properties config = new java.util.Properties(); config.put(STRICT_HOST_KEY_CHECKING, NO); session.setConfig(config); session.setUserInfo(userInfo); } }; SshSessionFactory.setInstance(sessionFactory); /* * Enable clone of https url by trusting those urls */ // Create a trust manager that does not validate certificate chains TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; } public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { } public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { } } }; final String https_proxy = System.getenv(HTTPS_PROXY); final String http_proxy = System.getenv(HTTP_PROXY); ProxySelector.setDefault(new ProxySelector() { final ProxySelector delegate = ProxySelector.getDefault(); @Override public List<Proxy> select(URI uri) { // Filter the URIs to be proxied if (uri.toString().contains(HTTPS) && StringUtils.isNotEmpty(http_proxy) && http_proxy != null) { try { URI httpsUri = new URI(https_proxy); String host = httpsUri.getHost(); int port = httpsUri.getPort(); return Arrays.asList(new Proxy(Type.HTTP, InetSocketAddress.createUnresolved(host, port))); } catch (URISyntaxException e) { if (debugEnabled) { S_LOGGER.debug("Url exception caught in https block of additionalAuthentication()"); } } } if (uri.toString().contains(HTTP) && StringUtils.isNotEmpty(http_proxy) && http_proxy != null) { try { URI httpUri = new URI(http_proxy); String host = httpUri.getHost(); int port = httpUri.getPort(); return Arrays.asList(new Proxy(Type.HTTP, InetSocketAddress.createUnresolved(host, port))); } catch (URISyntaxException e) { if (debugEnabled) { S_LOGGER.debug("Url exception caught in http block of additionalAuthentication()"); } } } // revert to the default behaviour return delegate == null ? Arrays.asList(Proxy.NO_PROXY) : delegate.select(uri); } @Override public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { if (uri == null || sa == null || ioe == null) { throw new IllegalArgumentException("Arguments can't be null."); } } }); // Install the all-trusting trust manager try { SSLContext sc = SSLContext.getInstance(SSL); sc.init(null, trustAllCerts, new java.security.SecureRandom()); HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); } catch (GeneralSecurityException e) { e.getLocalizedMessage(); } }
From source file:com.datatorrent.stram.engine.StreamingContainer.java
@SuppressWarnings("unchecked") private void deployInputStreams(List<OperatorDeployInfo> operatorList, HashMap<String, ComponentContextPair<Stream, StreamContext>> newStreams) throws UnknownHostException { /*// w w w. ja v a 2s . c om * collect any input operators along with their smallest window id, * those are subsequently used to setup the window generator */ ArrayList<OperatorDeployInfo> inputNodes = new ArrayList<OperatorDeployInfo>(); long smallestCheckpointedWindowId = Long.MAX_VALUE; //a simple map which maps the oio node to it's the node which owns the thread. Map<Integer, Integer> oioNodes = new ConcurrentHashMap<Integer, Integer>(); /* * Hook up all the downstream ports. There are 2 places where we deal with more than 1 * downstream ports. The first one follows immediately for WindowGenerator. The second * case is when source for the input port of some node in this container is in another * container. So we need to create the stream. We need to track this stream along with * other streams,and many such streams may exist, we hash them against buffer server * info as we did for outputs but throw in the sinkid in the mix as well. */ for (OperatorDeployInfo ndi : operatorList) { if (ndi.inputs == null || ndi.inputs.isEmpty()) { /* * This has to be InputNode, so let's hook the WindowGenerator to it. * A node which does not take any input cannot exist in the DAG since it would be completely * unaware of the windows. So for that reason, AbstractInputNode allows Component.INPUT port. */ inputNodes.add(ndi); /* * When we activate the window Generator, we plan to activate it only from required windowId. */ ndi.checkpoint = getFinishedCheckpoint(ndi); if (ndi.checkpoint.windowId < smallestCheckpointedWindowId) { smallestCheckpointedWindowId = ndi.checkpoint.windowId; } } else { Node<?> node = nodes.get(ndi.id); for (OperatorDeployInfo.InputDeployInfo nidi : ndi.inputs) { if (nidi.streamCodecs.size() != 1) { throw new IllegalStateException("Only one input codec configuration should be present"); } Map.Entry<Integer, StreamCodec<?>> entry = nidi.streamCodecs.entrySet().iterator().next(); Integer streamCodecIdentifier = entry.getKey(); StreamCodec<?> streamCodec = entry.getValue(); String sourceIdentifier = Integer.toString(nidi.sourceNodeId).concat(Component.CONCAT_SEPARATOR) .concat(nidi.sourcePortName); String sinkIdentifier = Integer.toString(ndi.id).concat(Component.CONCAT_SEPARATOR) .concat(nidi.portName); int queueCapacity = getValue(PortContext.QUEUE_CAPACITY, nidi, ndi); Checkpoint checkpoint = getFinishedCheckpoint(ndi); ComponentContextPair<Stream, StreamContext> pair = streams.get(sourceIdentifier); if (pair == null) { pair = newStreams.get(sourceIdentifier); } if (pair == null) { /* * We connect to the buffer server for the input on this port. * We have already placed all the output streams for all the operators in this container. * Yet, there is no stream which can source this port so it has to come from the buffer * server, so let's make a connection to it. */ assert (nidi.locality != Locality.CONTAINER_LOCAL && nidi.locality != Locality.THREAD_LOCAL); StreamContext context = new StreamContext(nidi.declaredStreamId); context.setBufferServerAddress( InetSocketAddress.createUnresolved(nidi.bufferServerHost, nidi.bufferServerPort)); InetAddress inetAddress = context.getBufferServerAddress().getAddress(); if (inetAddress != null && NetUtils.isLocalAddress(inetAddress)) { context.setBufferServerAddress( new InetSocketAddress(InetAddress.getByName(null), nidi.bufferServerPort)); } context.put(StreamContext.BUFFER_SERVER_TOKEN, nidi.bufferServerToken); String connIdentifier = sourceIdentifier + Component.CONCAT_SEPARATOR + streamCodecIdentifier; context.setPortId(nidi.portName); context.put(StreamContext.CODEC, streamCodec); context.put(StreamContext.EVENT_LOOP, eventloop); context.setPartitions(nidi.partitionMask, nidi.partitionKeys); //context.setSourceId(sourceIdentifier); context.setSourceId(connIdentifier); context.setSinkId(sinkIdentifier); context.setFinishedWindowId(checkpoint.windowId); BufferServerSubscriber subscriber = fastPublisherSubscriber ? new FastSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":") .concat(String.valueOf(nidi.bufferServerPort)).concat("/") .concat(connIdentifier), queueCapacity) : new BufferServerSubscriber("tcp://".concat(nidi.bufferServerHost).concat(":") .concat(String.valueOf(nidi.bufferServerPort)).concat("/") .concat(connIdentifier), queueCapacity); if (streamCodec instanceof StreamCodecWrapperForPersistance) { subscriber.acquireReservoirForPersistStream(sinkIdentifier, queueCapacity, streamCodec); } SweepableReservoir reservoir = subscriber.acquireReservoir(sinkIdentifier, queueCapacity); if (checkpoint.windowId >= 0) { node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, reservoir, checkpoint.windowId)); } node.connectInputPort(nidi.portName, reservoir); newStreams.put(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(subscriber, context)); logger.debug("put input stream {} against key {}", subscriber, sinkIdentifier); } else { assert (nidi.locality == Locality.CONTAINER_LOCAL || nidi.locality == Locality.THREAD_LOCAL); /* we are still dealing with the MuxStream originating at the output of the source port */ StreamContext inlineContext = new StreamContext(nidi.declaredStreamId); inlineContext.setSourceId(sourceIdentifier); inlineContext.setSinkId(sinkIdentifier); Stream stream; switch (nidi.locality) { case CONTAINER_LOCAL: int outputQueueCapacity = getOutputQueueCapacity(operatorList, nidi.sourceNodeId, nidi.sourcePortName); if (outputQueueCapacity > queueCapacity) { queueCapacity = outputQueueCapacity; } stream = new InlineStream(queueCapacity); if (checkpoint.windowId >= 0) { node.connectInputPort(nidi.portName, new WindowIdActivatedReservoir(sinkIdentifier, (SweepableReservoir) stream, checkpoint.windowId)); } break; case THREAD_LOCAL: stream = new OiOStream(); oioNodes.put(ndi.id, nidi.sourceNodeId); break; default: throw new IllegalStateException("Locality can be either ContainerLocal or ThreadLocal"); } node.connectInputPort(nidi.portName, (SweepableReservoir) stream); newStreams.put(sinkIdentifier, new ComponentContextPair<Stream, StreamContext>(stream, inlineContext)); if (!(pair.component instanceof Stream.MultiSinkCapableStream)) { String originalSinkId = pair.context.getSinkId(); /* we come here only if we are trying to augment the dag */ StreamContext muxContext = new StreamContext(nidi.declaredStreamId); muxContext.setSourceId(sourceIdentifier); muxContext.setFinishedWindowId(checkpoint.windowId); muxContext.setSinkId(originalSinkId); MuxStream muxStream = new MuxStream(); muxStream.setSink(originalSinkId, pair.component); streams.put(originalSinkId, pair); Node<?> sourceNode = nodes.get(nidi.sourceNodeId); sourceNode.connectOutputPort(nidi.sourcePortName, muxStream); newStreams.put(sourceIdentifier, pair = new ComponentContextPair<Stream, StreamContext>(muxStream, muxContext)); } /* here everything should be multisink capable */ if (streamCodec instanceof StreamCodecWrapperForPersistance) { PartitionAwareSinkForPersistence pas; if (nidi.partitionKeys == null) { pas = new PartitionAwareSinkForPersistence( (StreamCodecWrapperForPersistance<Object>) streamCodec, nidi.partitionMask, stream); } else { pas = new PartitionAwareSinkForPersistence( (StreamCodecWrapperForPersistance<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream); } ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas); } else if (nidi.partitionKeys == null || nidi.partitionKeys.isEmpty()) { ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, stream); } else { /* * generally speaking we do not have partitions on the inline streams so the control should not * come here but if it comes, then we are ready to handle it using the partition aware streams. */ PartitionAwareSink<Object> pas = new PartitionAwareSink<Object>( streamCodec == null ? nonSerializingStreamCodec : (StreamCodec<Object>) streamCodec, nidi.partitionKeys, nidi.partitionMask, stream); ((Stream.MultiSinkCapableStream) pair.component).setSink(sinkIdentifier, pas); } String streamSinkId = pair.context.getSinkId(); if (streamSinkId == null) { pair.context.setSinkId(sinkIdentifier); } else { pair.context.setSinkId(streamSinkId.concat(", ").concat(sinkIdentifier)); } } } } } setupOiOGroups(oioNodes); if (!inputNodes.isEmpty()) { WindowGenerator windowGenerator = setupWindowGenerator(smallestCheckpointedWindowId); for (OperatorDeployInfo ndi : inputNodes) { generators.put(ndi.id, windowGenerator); Node<?> node = nodes.get(ndi.id); SweepableReservoir reservoir = windowGenerator.acquireReservoir(String.valueOf(ndi.id), 1024); if (ndi.checkpoint.windowId >= 0) { node.connectInputPort(Node.INPUT, new WindowIdActivatedReservoir(Integer.toString(ndi.id), reservoir, ndi.checkpoint.windowId)); } node.connectInputPort(Node.INPUT, reservoir); } } }
From source file:com.dwdesign.tweetings.util.Utils.java
public static Proxy getProxy(final Context context) { if (context == null) return null; final SharedPreferences prefs = context.getSharedPreferences(SHARED_PREFERENCES_NAME, Context.MODE_PRIVATE); final boolean enable_proxy = prefs.getBoolean(PREFERENCE_KEY_ENABLE_PROXY, false); if (!enable_proxy) return Proxy.NO_PROXY; final String proxy_host = prefs.getString(PREFERENCE_KEY_PROXY_HOST, null); final int proxy_port = parseInt(prefs.getString(PREFERENCE_KEY_PROXY_PORT, "-1")); if (!isNullOrEmpty(proxy_host) && proxy_port > 0) { final SocketAddress addr = InetSocketAddress.createUnresolved(proxy_host, proxy_port); return new Proxy(Proxy.Type.HTTP, addr); }/*from ww w .ja v a 2 s . co m*/ return Proxy.NO_PROXY; }
From source file:com.datatorrent.stram.StreamingContainerManager.java
/** * process the heartbeat from each container. * called by the RPC thread for each container. (i.e. called by multiple threads) * * @param heartbeat//from w w w .j av a2s . com * @return heartbeat response */ @SuppressWarnings("StatementWithEmptyBody") public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) { long currentTimeMillis = clock.getTime(); final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId()); if (sca == null || sca.container.getState() == PTContainer.State.KILLED) { // could be orphaned container that was replaced and needs to terminate LOG.error("Unknown container {}", heartbeat.getContainerId()); ContainerHeartbeatResponse response = new ContainerHeartbeatResponse(); response.shutdown = true; return response; } //LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()}); if (sca.container.getState() == PTContainer.State.ALLOCATED) { // capture dynamically assigned address from container if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) { sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost, heartbeat.bufferServerPort); LOG.info("Container {} buffer server: {}", sca.container.getExternalId(), sca.container.bufferServerAddress); } final long containerStartTime = System.currentTimeMillis(); sca.container.setState(PTContainer.State.ACTIVE); sca.container.setStartedTime(containerStartTime); sca.container.setFinishedTime(-1); sca.jvmName = heartbeat.jvmName; poolExecutor.submit(new Runnable() { @Override public void run() { try { containerFile.append(sca.getContainerInfo()); } catch (IOException ex) { LOG.warn("Cannot write to container file"); } for (PTOperator ptOp : sca.container.getOperators()) { try { JSONObject operatorInfo = new JSONObject(); operatorInfo.put("name", ptOp.getName()); operatorInfo.put("id", ptOp.getId()); operatorInfo.put("container", sca.container.getExternalId()); operatorInfo.put("startTime", containerStartTime); operatorFile.append(operatorInfo); } catch (IOException | JSONException ex) { LOG.warn("Cannot write to operator file: ", ex); } } } }); } if (heartbeat.restartRequested) { LOG.error("Container {} restart request", sca.container.getExternalId()); containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId()); } sca.memoryMBFree = heartbeat.memoryMBFree; sca.gcCollectionCount = heartbeat.gcCollectionCount; sca.gcCollectionTime = heartbeat.gcCollectionTime; sca.undeployOpers.clear(); sca.deployOpers.clear(); if (!this.deployChangeInProgress.get()) { sca.deployCnt = this.deployChangeCnt; } Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size()); for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) { long maxEndWindowTimestamp = 0; reportedOperators.add(shb.nodeId); PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId()); if (oper == null) { LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(), heartbeat.getContainerId()); sca.undeployOpers.add(shb.nodeId); continue; } if (shb.requestResponse != null) { for (StatsListener.OperatorResponse obj : shb.requestResponse) { if (obj instanceof OperatorResponse) { // This is to identify platform requests commandResponse.put((Long) obj.getResponseId(), obj.getResponse()); LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId()); } else { // This is to identify user requests oper.stats.responses.add(obj); } } } //LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId()); if (!(oper.getState() == PTOperator.State.ACTIVE && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) { // deploy state may require synchronization processOperatorDeployStatus(oper, shb, sca); } oper.stats.lastHeartbeat = shb; List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer(); if (!statsList.isEmpty()) { long tuplesProcessed = 0; long tuplesEmitted = 0; long totalCpuTimeUsed = 0; int statCount = 0; long maxDequeueTimestamp = -1; oper.stats.recordingId = null; final OperatorStatus status = oper.stats; status.statsRevs.checkout(); for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) { entry.getValue().recordingId = null; } for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) { entry.getValue().recordingId = null; } for (ContainerStats.OperatorStats stats : statsList) { if (stats == null) { LOG.warn("Operator {} statistics list contains null element", shb.getNodeId()); continue; } /* report checkpoint-ed WindowId status of the operator */ if (stats.checkpoint instanceof Checkpoint) { if (oper.getRecentCheckpoint() == null || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) { addCheckpoint(oper, (Checkpoint) stats.checkpoint); if (stats.checkpointStats != null) { status.checkpointStats = stats.checkpointStats; status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime); } oper.failureCount = 0; } } oper.stats.recordingId = stats.recordingId; /* report all the other stuff */ // calculate the stats related to end window EndWindowStats endWindowStats = new EndWindowStats(); // end window stats for a particular window id for a particular node Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts; if (ports != null) { Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size()); for (ContainerStats.OperatorStats.PortStats s : ports) { currentInputPortSet.add(s.id); PortStatus ps = status.inputPortStatusList.get(s.id); if (ps == null) { ps = status.new PortStatus(); ps.portName = s.id; status.inputPortStatusList.put(s.id, ps); } ps.totalTuples += s.tupleCount; ps.recordingId = s.recordingId; tuplesProcessed += s.tupleCount; endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp); Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id); Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName); if (lastEndWindowTimestamp == null) { lastEndWindowTimestamp = lastStatsTimestamp; } long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0); //LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp); ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis); ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis); ps.queueSizeMA.add(s.queueSize); operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp); if (maxEndWindowTimestamp < s.endWindowTimestamp) { maxEndWindowTimestamp = s.endWindowTimestamp; } if (s.endWindowTimestamp > maxDequeueTimestamp) { maxDequeueTimestamp = s.endWindowTimestamp; } } // need to remove dead ports, for unifiers Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet() .iterator(); while (it.hasNext()) { Map.Entry<String, PortStatus> entry = it.next(); if (!currentInputPortSet.contains(entry.getKey())) { it.remove(); } } } ports = stats.outputPorts; if (ports != null) { Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size()); for (ContainerStats.OperatorStats.PortStats s : ports) { currentOutputPortSet.add(s.id); PortStatus ps = status.outputPortStatusList.get(s.id); if (ps == null) { ps = status.new PortStatus(); ps.portName = s.id; status.outputPortStatusList.put(s.id, ps); } ps.totalTuples += s.tupleCount; ps.recordingId = s.recordingId; tuplesEmitted += s.tupleCount; Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id); Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName); if (lastEndWindowTimestamp == null) { lastEndWindowTimestamp = lastStatsTimestamp; } long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0); //LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp); ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis); ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis); operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp); if (maxEndWindowTimestamp < s.endWindowTimestamp) { maxEndWindowTimestamp = s.endWindowTimestamp; } } if (ports.size() > 0) { endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp; } // need to remove dead ports, for unifiers Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet() .iterator(); while (it.hasNext()) { Map.Entry<String, PortStatus> entry = it.next(); if (!currentOutputPortSet.contains(entry.getKey())) { it.remove(); } } } // for output operator, just take the maximum dequeue time for emit timestamp. // (we don't know the latency for output operators because they don't emit tuples) if (endWindowStats.emitTimestamp < 0) { endWindowStats.emitTimestamp = maxDequeueTimestamp; } if (status.currentWindowId.get() != stats.windowId) { status.lastWindowIdChangeTms = currentTimeMillis; status.currentWindowId.set(stats.windowId); } totalCpuTimeUsed += stats.cpuTimeUsed; statCount++; if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) { endWindowStats.counters = stats.counters; } if (oper.getOperatorMeta().getMetricAggregatorMeta() != null && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) { endWindowStats.metrics = stats.metrics; } if (stats.windowId > currentEndWindowStatsWindowId) { Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap .get(stats.windowId); if (endWindowStatsMap == null) { endWindowStatsMap = new ConcurrentSkipListMap<Integer, EndWindowStats>(); Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap .putIfAbsent(stats.windowId, endWindowStatsMap); if (endWindowStatsMapPrevious != null) { endWindowStatsMap = endWindowStatsMapPrevious; } } endWindowStatsMap.put(shb.getNodeId(), endWindowStats); if (!oper.getInputs().isEmpty()) { long latency = Long.MAX_VALUE; long adjustedEndWindowEmitTimestamp = endWindowStats.emitTimestamp; MovingAverageLong rpcLatency = rpcLatencies.get(oper.getContainer().getExternalId()); if (rpcLatency != null) { adjustedEndWindowEmitTimestamp += rpcLatency.getAvg(); } PTOperator slowestUpstream = null; for (PTInput input : oper.getInputs()) { PTOperator upstreamOp = input.source.source; if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) { continue; } EndWindowStats ews = endWindowStatsMap.get(upstreamOp.getId()); long portLatency; if (ews == null) { // This is when the operator is likely to be behind too many windows. We need to give an estimate for // latency at this point, by looking at the number of windows behind int widthMillis = plan.getLogicalPlan() .getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS); portLatency = (upstreamOp.stats.currentWindowId.get() - oper.stats.currentWindowId.get()) * widthMillis; } else { MovingAverageLong upstreamRPCLatency = rpcLatencies .get(upstreamOp.getContainer().getExternalId()); portLatency = adjustedEndWindowEmitTimestamp - ews.emitTimestamp; if (upstreamRPCLatency != null) { portLatency -= upstreamRPCLatency.getAvg(); } } if (portLatency < 0) { portLatency = 0; } if (latency > portLatency) { latency = portLatency; slowestUpstream = upstreamOp; } } status.latencyMA.add(latency); slowestUpstreamOp.put(oper, slowestUpstream); } Set<Integer> allCurrentOperators = plan.getAllOperators().keySet(); int numOperators = plan.getAllOperators().size(); if (allCurrentOperators.containsAll(endWindowStatsMap.keySet()) && endWindowStatsMap.size() == numOperators) { completeEndWindowStatsWindowId = stats.windowId; } } } status.totalTuplesProcessed.add(tuplesProcessed); status.totalTuplesEmitted.add(tuplesEmitted); OperatorMeta logicalOperator = oper.getOperatorMeta(); LogicalOperatorStatus logicalStatus = logicalOperator.getStatus(); if (!oper.isUnifier()) { logicalStatus.totalTuplesProcessed += tuplesProcessed; logicalStatus.totalTuplesEmitted += tuplesEmitted; } long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId()) ? operatorLastEndWindowTimestamps.get(oper.getId()) : lastStatsTimestamp; if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) { double tuplesProcessedPMSMA = 0.0; double tuplesEmittedPMSMA = 0.0; if (statCount != 0) { //LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp); status.cpuNanosPMSMA.add(totalCpuTimeUsed, maxEndWindowTimestamp - lastMaxEndWindowTimestamp); } for (PortStatus ps : status.inputPortStatusList.values()) { tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg(); } for (PortStatus ps : status.outputPortStatusList.values()) { tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg(); } status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000)); status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000)); } else { //LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(), maxEndWindowTimestamp, lastMaxEndWindowTimestamp); } operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp); status.listenerStats.add(statsList); this.reportStats.put(oper, oper); status.statsRevs.commit(); } if (lastStatsTimestamp < maxEndWindowTimestamp) { lastStatsTimestamp = maxEndWindowTimestamp; } } sca.lastHeartbeatMillis = currentTimeMillis; for (PTOperator oper : sca.container.getOperators()) { if (!reportedOperators.contains(oper.getId())) { processOperatorDeployStatus(oper, null, sca); } } ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca); if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) { LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId()); rsp.shutdown = true; } else { if (sca.shutdownRequested) { LOG.info("requesting shutdown for container {}", heartbeat.getContainerId()); rsp.shutdown = true; } } List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests : new ArrayList<StramToNodeRequest>(); ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests(); while (true) { StramToNodeRequest r = operatorRequests.poll(); if (r == null) { break; } requests.add(r); } rsp.nodeRequests = requests; rsp.committedWindowId = committedWindowId; return rsp; }
From source file:org.mariotaku.twidere.util.Utils.java
public static Proxy getProxy(final Context context) { if (context == null) return null; final SharedPreferences prefs = context.getSharedPreferences(SHARED_PREFERENCES_NAME, Context.MODE_PRIVATE); final boolean enable_proxy = prefs.getBoolean(PREFERENCE_KEY_ENABLE_PROXY, false); if (!enable_proxy) return Proxy.NO_PROXY; final String proxy_host = prefs.getString(PREFERENCE_KEY_PROXY_HOST, null); final int proxy_port = parseInt(prefs.getString(PREFERENCE_KEY_PROXY_PORT, "-1")); if (!isEmpty(proxy_host) && proxy_port > 0) { final SocketAddress addr = InetSocketAddress.createUnresolved(proxy_host, proxy_port); return new Proxy(Proxy.Type.HTTP, addr); }/*from w w w . j a v a2 s. c om*/ return Proxy.NO_PROXY; }