List of usage examples for org.apache.hadoop.net NetUtils createSocketAddrForHost
public static InetSocketAddress createSocketAddrForHost(String host, int port)
From source file:com.datatorrent.stram.RecoverableRpcProxy.java
License:Apache License
private void connect() throws IOException { FSRecoveryHandler fsrh = new FSRecoveryHandler(appPath, conf); String uriStr = fsrh.readConnectUri(); if (!uriStr.equals(lastConnectURI)) { // reset timeout LOG.debug("Got new RPC connect address {}", uriStr); lastCompletedCallTms = System.currentTimeMillis(); lastConnectURI = uriStr;//w ww. j ava 2 s. c o m } URI heartbeatUri = URI.create(uriStr); String queryStr = heartbeatUri.getQuery(); List<NameValuePair> queryList = null; if (queryStr != null) { queryList = URLEncodedUtils.parse(queryStr, Charset.defaultCharset()); } if (queryList != null) { for (NameValuePair pair : queryList) { String value = pair.getValue(); String key = pair.getName(); if (QP_rpcTimeout.equals(key)) { this.rpcTimeout = Integer.parseInt(value); } else if (QP_retryTimeoutMillis.equals(key)) { this.retryTimeoutMillis = Long.parseLong(value); } else if (QP_retryDelayMillis.equals(key)) { this.retryDelayMillis = Long.parseLong(value); } } } InetSocketAddress address = NetUtils.createSocketAddrForHost(heartbeatUri.getHost(), heartbeatUri.getPort()); umbilical = RPC.getProxy(StreamingContainerUmbilicalProtocol.class, StreamingContainerUmbilicalProtocol.versionID, address, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout); }
From source file:io.hops.util.DBUtility.java
License:Apache License
public static RMNode processHopRMNodeCompsForScheduler(RMNodeComps hopRMNodeComps, RMContext rmContext) throws InvalidProtocolBufferException { org.apache.hadoop.yarn.api.records.NodeId nodeId; RMNode rmNode = null;// www.j av a 2 s. c o m if (hopRMNodeComps != null) { nodeId = ConverterUtils.toNodeId(hopRMNodeComps.getRMNodeId()); rmNode = rmContext.getRMNodes().get(nodeId); // The first time we are receiving the RMNode, this will happen when the node registers if (rmNode == null) { // Retrieve heartbeat boolean nextHeartbeat = true; // Create Resource Resource resource = null; if (hopRMNodeComps.getHopResource() != null) { resource = Resource.newInstance(hopRMNodeComps.getHopResource().getMemory(), hopRMNodeComps.getHopResource().getVirtualCores()); } else { LOG.error("ResourceOption should not be null"); resource = Resource.newInstance(0, 0); } /*rmNode = new RMNodeImplDist(nodeId, rmContext, hopRMNodeComps.getHopRMNode().getHostName(), hopRMNodeComps.getHopRMNode().getCommandPort(), hopRMNodeComps.getHopRMNode().getHttpPort(), ResourceTrackerService.resolve(hopRMNodeComps.getHopRMNode().getHostName()), resourceOption, hopRMNodeComps.getHopRMNode().getNodemanagerVersion(), hopRMNodeComps.getHopRMNode().getHealthReport(), hopRMNodeComps.getHopRMNode().getLastHealthReportTime(), nextHeartbeat);*/ rmNode = new RMNodeImplDist(nodeId, rmContext, hopRMNodeComps.getHopRMNode().getHostName(), hopRMNodeComps.getHopRMNode().getCommandPort(), hopRMNodeComps.getHopRMNode().getHttpPort(), ResourceTrackerService.resolve(hopRMNodeComps.getHopRMNode().getHostName()), resource, hopRMNodeComps.getHopRMNode().getNodemanagerVersion()); // Force Java to put the host in cache NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); } // Update the RMNode if (hopRMNodeComps.getHopRMNode() != null) { ((RMNodeImplDist) rmNode).setState(hopRMNodeComps.getHopRMNode().getCurrentState()); } if (hopRMNodeComps.getHopUpdatedContainerInfo() != null) { List<io.hops.metadata.yarn.entity.UpdatedContainerInfo> hopUpdatedContainerInfoList = hopRMNodeComps .getHopUpdatedContainerInfo(); if (hopUpdatedContainerInfoList != null && !hopUpdatedContainerInfoList.isEmpty()) { ConcurrentLinkedQueue<org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> updatedContainerInfoQueue = new ConcurrentLinkedQueue<>(); Map<Integer, org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> ucis = new HashMap<>(); LOG.debug(hopRMNodeComps.getRMNodeId() + " getting ucis " + hopUpdatedContainerInfoList.size() + " pending event " + hopRMNodeComps.getPendingEvent().getId().getEventId()); for (io.hops.metadata.yarn.entity.UpdatedContainerInfo hopUCI : hopUpdatedContainerInfoList) { if (!ucis.containsKey(hopUCI.getUpdatedContainerInfoId())) { ucis.put(hopUCI.getUpdatedContainerInfoId(), new org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo( new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(), new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(), hopUCI.getUpdatedContainerInfoId())); } ContainerId cid = ConverterUtils.toContainerId(hopUCI.getContainerId()); io.hops.metadata.yarn.entity.ContainerStatus hopContainerStatus = hopRMNodeComps .getHopContainersStatusMap().get(hopUCI.getContainerId()); org.apache.hadoop.yarn.api.records.ContainerStatus conStatus = org.apache.hadoop.yarn.api.records.ContainerStatus .newInstance(cid, ContainerState.valueOf(hopContainerStatus.getState()), hopContainerStatus.getDiagnostics(), hopContainerStatus.getExitstatus()); // Check ContainerStatus state to add it in the appropriate list if (conStatus != null) { LOG.debug("add uci for container " + conStatus.getContainerId() + " status " + conStatus.getState()); if (conStatus.getState().equals(ContainerState.RUNNING)) { ucis.get(hopUCI.getUpdatedContainerInfoId()).getNewlyLaunchedContainers() .add(conStatus); } else if (conStatus.getState().equals(ContainerState.COMPLETE)) { ucis.get(hopUCI.getUpdatedContainerInfoId()).getCompletedContainers() .add(conStatus); } } } for (org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo uci : ucis .values()) { updatedContainerInfoQueue.add(uci); } ((RMNodeImplDist) rmNode).setUpdatedContainerInfo(updatedContainerInfoQueue); } else { LOG.debug(hopRMNodeComps.getRMNodeId() + " hopUpdatedContainerInfoList = null || hopUpdatedContainerInfoList.isEmpty() " + hopRMNodeComps.getPendingEvent().getId().getEventId()); } } else { LOG.debug(hopRMNodeComps.getRMNodeId() + " hopRMNodeFull.getHopUpdatedContainerInfo()=null " + hopRMNodeComps.getPendingEvent().getId().getEventId()); } } return rmNode; }
From source file:org.apache.hoya.yarn.appmaster.rpc.RpcBinder.java
License:Apache License
public static HoyaClusterProtocol getProxy(final Configuration conf, ApplicationReport application, final int rpcTimeout) throws IOException, HoyaException, InterruptedException { String host = application.getHost(); int port = application.getRpcPort(); String address = host + ":" + port; if (host == null || 0 == port) { throw new HoyaException(HoyaExitCodes.EXIT_CONNECTIVITY_PROBLEM, "Hoya YARN instance " + application.getName() + " isn't providing a valid address for the" + " Hoya RPC protocol: " + address); }//from w w w. ja v a2 s.co m UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(currentUser.getUserName()); final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort()); HoyaClusterProtocol realProxy; log.debug("Connecting to {}", serviceAddr); if (UserGroupInformation.isSecurityEnabled()) { org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken(); Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr); newUgi.addToken(token); realProxy = newUgi.doAs(new PrivilegedExceptionAction<HoyaClusterProtocol>() { @Override public HoyaClusterProtocol run() throws IOException { return connectToServer(serviceAddr, newUgi, conf, rpcTimeout); } }); } else { return connectToServer(serviceAddr, newUgi, conf, rpcTimeout); } return realProxy; }
From source file:org.apache.slider.server.appmaster.rpc.RpcBinder.java
License:Apache License
public static SliderClusterProtocol getProxy(final Configuration conf, ApplicationReport application, final int rpcTimeout) throws IOException, SliderException, InterruptedException { String host = application.getHost(); int port = application.getRpcPort(); String address = host + ":" + port; if (host == null || 0 == port) { throw new SliderException(SliderExitCodes.EXIT_CONNECTIVITY_PROBLEM, "Slider instance " + application.getName() + " isn't providing a valid address for the" + " Slider RPC protocol: " + address); }/*from ww w . ja v a2s . com*/ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(currentUser.getUserName()); final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort()); SliderClusterProtocol realProxy; log.debug("Connecting to {}", serviceAddr); if (UserGroupInformation.isSecurityEnabled()) { org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken(); Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr); newUgi.addToken(token); realProxy = newUgi.doAs(new PrivilegedExceptionAction<SliderClusterProtocol>() { @Override public SliderClusterProtocol run() throws IOException { return connectToServer(serviceAddr, newUgi, conf, rpcTimeout); } }); } else { return connectToServer(serviceAddr, newUgi, conf, rpcTimeout); } return realProxy; }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
@Private public static DAGClientAMProtocolBlockingPB getAMProxy(final Configuration conf, String amHost, int amRpcPort, org.apache.hadoop.yarn.api.records.Token clientToAMToken) throws IOException { final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(amHost, amRpcPort); UserGroupInformation userUgi = UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()); if (clientToAMToken != null) { Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr); userUgi.addToken(token);/*from ww w .j a va 2 s . c o m*/ } if (LOG.isDebugEnabled()) { LOG.debug("Connecting to Tez AM at " + serviceAddr); } DAGClientAMProtocolBlockingPB proxy = null; try { proxy = userUgi.doAs(new PrivilegedExceptionAction<DAGClientAMProtocolBlockingPB>() { @Override public DAGClientAMProtocolBlockingPB run() throws IOException { RPC.setProtocolEngine(conf, DAGClientAMProtocolBlockingPB.class, ProtobufRpcEngine.class); return (DAGClientAMProtocolBlockingPB) RPC.getProxy(DAGClientAMProtocolBlockingPB.class, 0, serviceAddr, conf); } }); } catch (InterruptedException e) { throw new IOException("Failed to connect to AM", e); } return proxy; }
From source file:org.apache.tez.dag.api.client.DAGClientServer.java
License:Apache License
@Override public void serviceStart() { try {/* w w w .j a va 2s . com*/ Configuration conf = getConfig(); InetSocketAddress addr = new InetSocketAddress(0); DAGClientAMProtocolBlockingPBServerImpl service = new DAGClientAMProtocolBlockingPBServerImpl( realInstance); BlockingService blockingService = DAGClientAMProtocol.newReflectiveBlockingService(service); int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT, TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT); server = createServer(DAGClientAMProtocolBlockingPB.class, addr, conf, numHandlers, blockingService, TezConfiguration.TEZ_AM_CLIENT_AM_PORT_RANGE); // Enable service authorization? if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new TezAMPolicyProvider()); } server.start(); InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server); this.bindAddress = NetUtils.createSocketAddrForHost( serverBindAddress.getAddress().getCanonicalHostName(), serverBindAddress.getPort()); LOG.info("Instantiated DAGClientRPCServer at " + bindAddress); } catch (Exception e) { LOG.error("Failed to start DAGClientServer: ", e); throw new TezUncheckedException(e); } }
From source file:org.apache.tez.dag.app.taskcomm.TezTestServiceTaskCommunicatorWithErrors.java
License:Apache License
@Override public InetSocketAddress getAddress() { return NetUtils.createSocketAddrForHost("localhost", 0); }
From source file:org.apache.tez.dag.app.TezTaskCommunicatorImpl.java
License:Apache License
protected void startRpcServer() { try {//from www.j ava 2 s . c o m JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); jobTokenSecretManager.addTokenForJob(tokenIdentifier, sessionToken); server = new RPC.Builder(conf).setProtocol(TezTaskUmbilicalProtocol.class).setBindAddress("0.0.0.0") .setPort(0).setInstance(taskUmbilical) .setNumHandlers(conf.getInt(TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT, TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT)) .setPortRangeConfig(TezConfiguration.TEZ_AM_TASK_AM_PORT_RANGE) .setSecretManager(jobTokenSecretManager).build(); // Enable service authorization? if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new TezAMPolicyProvider()); } server.start(); InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server); this.address = NetUtils.createSocketAddrForHost(serverBindAddress.getAddress().getCanonicalHostName(), serverBindAddress.getPort()); LOG.info("Instantiated TezTaskCommunicator RPC at " + this.address); } catch (IOException e) { throw new TezUncheckedException(e); } }
From source file:org.apache.tez.runtime.task.TezChild.java
License:Apache License
public TezChild(Configuration conf, String host, int port, String containerIdentifier, String tokenIdentifier, int appAttemptNumber, String workingDir, String[] localDirs, Map<String, String> serviceProviderEnvMap, ObjectRegistryImpl objectRegistry, String pid, ExecutionContext executionContext, Credentials credentials, long memAvailable, String user) throws IOException, InterruptedException { this.defaultConf = conf; this.containerIdString = containerIdentifier; this.appAttemptNumber = appAttemptNumber; this.localDirs = localDirs; this.serviceProviderEnvMap = serviceProviderEnvMap; this.workingDir = workingDir; this.pid = pid; this.executionContext = executionContext; this.credentials = credentials; this.memAvailable = memAvailable; this.user = user; getTaskMaxSleepTime = defaultConf.getInt(TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX, TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT); amHeartbeatInterval = defaultConf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT); sendCounterInterval = defaultConf.getLong(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT); maxEventsToGet = defaultConf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT, TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT); ExecutorService executor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TezChild").build()); this.executor = MoreExecutors.listeningDecorator(executor); this.objectRegistry = objectRegistry; if (LOG.isDebugEnabled()) { LOG.debug("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.debug(token);//from w w w. ja v a 2 s . co m } } this.isLocal = defaultConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT); UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(tokenIdentifier); Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, TezCommonUtils.convertJobTokenToBytes(jobToken)); if (!isLocal) { final InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port); SecurityUtil.setTokenService(jobToken, address); taskOwner.addToken(jobToken); umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() { @Override public TezTaskUmbilicalProtocol run() throws Exception { return RPC.getProxy(TezTaskUmbilicalProtocol.class, TezTaskUmbilicalProtocol.versionID, address, defaultConf); } }); } }
From source file:org.apache.tez.service.impl.TezTestServiceProtocolServerImpl.java
License:Apache License
@Override public void serviceStart() { Configuration conf = getConfig(); int numHandlers = 3; InetSocketAddress addr = new InetSocketAddress(0); try {/*www. ja v a 2s .co m*/ server = createServer(TezTestServiceProtocolBlockingPB.class, addr, conf, numHandlers, TezTestServiceProtocolProtos.TezTestServiceProtocol.newReflectiveBlockingService(this)); server.start(); } catch (IOException e) { LOG.error("Failed to run RPC Server", e); throw new RuntimeException(e); } InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server); this.bindAddress.set(NetUtils.createSocketAddrForHost(serverBindAddress.getAddress().getCanonicalHostName(), serverBindAddress.getPort())); LOG.info("Instantiated TestTestServiceListener at " + bindAddress); }