List of usage examples for org.apache.hadoop.yarn.ipc YarnRPC create
public static YarnRPC create(Configuration conf)
From source file:com.twitter.hraven.hadoopJobMonitor.rpc.ClientCache.java
License:Apache License
protected MRClientProtocol instantiateHistoryProxy() throws IOException { final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); if (StringUtils.isEmpty(serviceAddr)) { return null; }//from w ww . j a v a2s . com LOG.debug("Connecting to HistoryServer at: " + serviceAddr); final YarnRPC rpc = YarnRPC.create(conf); LOG.debug("Connected to HistoryServer at: " + serviceAddr); UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() { @Override public MRClientProtocol run() { return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); } }); }
From source file:edu.uci.ics.hyracks.yarn.common.protocols.amrm.AMRMConnection.java
License:Apache License
public AMRMConnection(YarnConfiguration config) { this.config = config; Map<String, String> envs = System.getenv(); String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV); if (containerIdString == null) { throw new IllegalArgumentException("ContainerId not set in the environment"); }//from w w w . ja v a 2s . com ContainerId containerId = ConverterUtils.toContainerId(containerIdString); appAttemptId = containerId.getApplicationAttemptId(); InetSocketAddress rmAddress = NetUtils.createSocketAddr( config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); YarnRPC rpc = YarnRPC.create(config); amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config); }
From source file:edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection.java
License:Apache License
public YarnClientRMConnection(YarnConfiguration config) { this.config = config; InetSocketAddress remoteAddress = NetUtils .createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); Configuration appsManagerServerConf = new Configuration(config); appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER, ClientRMSecurityInfo.class, SecurityInfo.class); YarnRPC rpc = YarnRPC.create(appsManagerServerConf); crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf)); }
From source file:gobblin.hadoop.token.TokenUtils.java
License:Open Source License
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null;/*from www.j a v a 2s.co m*/ try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + jhToken.getIdentifier()); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
From source file:gobblin.util.hadoop.TokenUtils.java
License:Apache License
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null;/*www. ja v a 2 s.c om*/ try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier())); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
From source file:io.hops.util.GroupMembershipService.java
License:Apache License
protected void startServer() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); this.server = rpc.getServer(GroupMembership.class, this, groupMembershipServiceAddress, conf, null, conf.getInt(YarnConfiguration.RM_GROUP_MEMBERSHIP_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_GROUP_MEMBERSHIP_CLIENT_THREAD_COUNT)); this.server.start(); }
From source file:org.apache.hama.bsp.BSPApplicationMaster.java
License:Apache License
private BSPApplicationMaster(String[] args) throws Exception { if (args.length != 1) { throw new IllegalArgumentException(); }//from w w w . java 2 s. c o m this.jobFile = args[0]; this.localConf = new YarnConfiguration(); this.jobConf = getSubmitConfiguration(jobFile); fs = FileSystem.get(jobConf); this.applicationName = jobConf.get("bsp.job.name", "<no bsp job name defined>"); if (applicationName.isEmpty()) { this.applicationName = "<no bsp job name defined>"; } this.appAttemptId = getApplicationAttemptId(); this.yarnRPC = YarnRPC.create(localConf); this.clock = new SystemClock(); this.startTime = clock.getTime(); this.jobId = new BSPJobID(appAttemptId.toString(), 0); this.hostname = BSPNetUtils.getCanonicalHostname(); this.clientPort = BSPNetUtils.getFreePort(12000); // start our synchronization service startSyncServer(); startRPCServers(); /* * Make sure that this executes after the start the RPC servers, because we * are readjusting the configuration. */ rewriteSubmitConfiguration(jobFile, jobConf); String jobSplit = jobConf.get("bsp.job.split.file"); splits = null; if (jobSplit != null) { DataInputStream splitFile = fs.open(new Path(jobSplit)); try { splits = BSPJobClient.readSplitFile(splitFile); } finally { splitFile.close(); } } this.amrmRPC = getYarnRPCConnection(localConf); registerApplicationMaster(amrmRPC, hostname, clientPort, "http://localhost:8080"); }
From source file:org.apache.hama.bsp.YARNBSPJob.java
License:Apache License
public YARNBSPJob(HamaConfiguration conf) throws IOException { submitClient = new YARNBSPJobClient(conf); YarnConfiguration yarnConf = new YarnConfiguration(conf); this.rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = NetUtils .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); LOG.info("Connecting to ResourceManager at " + rmAddress); this.applicationsManager = ((ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, conf));/*w w w. j a va 2s. c o m*/ }
From source file:org.apache.hoya.yarn.appmaster.HoyaAppMaster.java
License:Apache License
/** * Create and run the cluster./*from w w w .j a v a 2 s . c o m*/ * @return exit code * @throws Throwable on a failure */ private int createAndRunCluster(String clustername) throws Throwable { HoyaVersionInfo.loadAndPrintVersionInfo(log); //load the cluster description from the cd argument String hoyaClusterDir = serviceArgs.getHoyaClusterURI(); URI hoyaClusterURI = new URI(hoyaClusterDir); Path clusterDirPath = new Path(hoyaClusterURI); HoyaFileSystem fs = getClusterFS(); // build up information about the running application -this // will be passed down to the cluster status MapOperations appInformation = new MapOperations(); AggregateConf instanceDefinition = InstanceIO.loadInstanceDefinitionUnresolved(fs, clusterDirPath); log.info("Deploying cluster {}:", instanceDefinition); //REVISIT: why is this done? appState.updateInstanceDefinition(instanceDefinition); File confDir = getLocalConfDir(); if (!confDir.exists() || !confDir.isDirectory()) { log.error("Bad conf dir {}", confDir); File parentFile = confDir.getParentFile(); log.error("Parent dir {}:\n{}", parentFile, HoyaUtils.listDir(parentFile)); throw new BadCommandArgumentsException("Configuration directory %s doesn't exist", confDir); } Configuration serviceConf = getConfig(); // Try to get the proper filtering of static resources through the yarn proxy working serviceConf.set("hadoop.http.filter.initializers", "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer"); conf = new YarnConfiguration(serviceConf); //get our provider MapOperations globalOptions = instanceDefinition.getInternalOperations().getGlobalOptions(); String providerType = globalOptions.getMandatoryOption(OptionKeys.INTERNAL_PROVIDER_NAME); log.info("Cluster provider type is {}", providerType); HoyaProviderFactory factory = HoyaProviderFactory.createHoyaProviderFactory(providerType); providerService = factory.createServerProvider(); // init the provider BUT DO NOT START IT YET providerService.init(getConfig()); addService(providerService); InetSocketAddress address = HoyaUtils.getRmSchedulerAddress(conf); log.info("RM is at {}", address); yarnRPC = YarnRPC.create(conf); /* * Extract the container ID. This is then * turned into an (incompete) container */ appMasterContainerID = ConverterUtils.toContainerId( HoyaUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name())); appAttemptID = appMasterContainerID.getApplicationAttemptId(); ApplicationId appid = appAttemptID.getApplicationId(); log.info("Hoya AM for ID {}", appid.getId()); appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString()); appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString()); appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString()); UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); Credentials credentials = currentUser.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); dob.close(); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); log.info("Token {}", token.getKind()); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // set up secret manager secretManager = new ClientToAMTokenSecretManager(appAttemptID, null); // if not a secure cluster, extract the username -it will be // propagated to workers if (!UserGroupInformation.isSecurityEnabled()) { hoyaUsername = System.getenv(HADOOP_USER_NAME); log.info(HADOOP_USER_NAME + "='{}'", hoyaUsername); } Map<String, String> envVars; /** * It is critical this section is synchronized, to stop async AM events * arriving while registering a restarting AM. */ synchronized (appState) { int heartbeatInterval = HEARTBEAT_INTERVAL; //add the RM client -this brings the callbacks in asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, this); addService(asyncRMClient); //wrap it for the app state model rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient); //now bring it up runChildService(asyncRMClient); //nmclient relays callbacks back to this class nmClientAsync = new NMClientAsyncImpl("nmclient", this); runChildService(nmClientAsync); //bring up the Hoya RPC service startHoyaRPCServer(); InetSocketAddress rpcServiceAddr = rpcService.getConnectAddress(); appMasterHostname = rpcServiceAddr.getHostName(); appMasterRpcPort = rpcServiceAddr.getPort(); appMasterTrackingUrl = null; log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort); appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname); appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort); //build the role map List<ProviderRole> providerRoles = new ArrayList<ProviderRole>(providerService.getRoles()); providerRoles.addAll(HoyaAMClientProvider.ROLES); // Start up the WebApp and track the URL for it webApp = new HoyaAMWebApp(); WebApps.$for("hoyaam", WebAppApi.class, new WebAppApiImpl(this, appState, providerService), "ws") .with(serviceConf).start(webApp); appMasterTrackingUrl = "http://" + appMasterHostname + ":" + webApp.port(); WebAppService<HoyaAMWebApp> webAppService = new WebAppService<HoyaAMWebApp>("hoya", webApp); webAppService.init(conf); webAppService.start(); addService(webAppService); appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/"); appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webApp.port()); // Register self with ResourceManager // This will start heartbeating to the RM // address = HoyaUtils.getRmSchedulerAddress(asyncRMClient.getConfig()); log.info("Connecting to RM at {},address tracking URL={}", appMasterRpcPort, appMasterTrackingUrl); RegisterApplicationMasterResponse response = asyncRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); Resource maxResources = response.getMaximumResourceCapability(); containerMaxMemory = maxResources.getMemory(); containerMaxCores = maxResources.getVirtualCores(); appState.setContainerLimits(maxResources.getMemory(), maxResources.getVirtualCores()); // set the RM-defined maximum cluster values appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores)); appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory)); boolean securityEnabled = UserGroupInformation.isSecurityEnabled(); if (securityEnabled) { secretManager.setMasterKey(response.getClientToAMTokenMasterKey().array()); applicationACLs = response.getApplicationACLs(); //tell the server what the ACLs are rpcService.getServer().refreshServiceAcl(conf, new HoyaAMPolicyProvider()); } // extract container list List<Container> liveContainers = AMRestartSupport.retrieveContainersFromPreviousAttempt(response); String amRestartSupported = Boolean.toString(liveContainers != null); appInformation.put(StatusKeys.INFO_AM_RESTART_SUPPORTED, amRestartSupported); //now validate the installation Configuration providerConf = providerService.loadProviderConfigurationInformation(confDir); providerService.validateApplicationConfiguration(instanceDefinition, confDir, securityEnabled); //determine the location for the role history data Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME); //build the instance appState.buildInstance(instanceDefinition, providerConf, providerRoles, fs.getFileSystem(), historyDir, liveContainers, appInformation); // add the AM to the list of nodes in the cluster appState.buildAppMasterNode(appMasterContainerID, appMasterHostname, webApp.port(), appMasterHostname + ":" + webApp.port()); // build up environment variables that the AM wants set in every container // irrespective of provider and role. envVars = new HashMap<String, String>(); if (hoyaUsername != null) { envVars.put(HADOOP_USER_NAME, hoyaUsername); } } String rolesTmpSubdir = appMasterContainerID.toString() + "/roles"; String amTmpDir = globalOptions.getMandatoryOption(OptionKeys.INTERNAL_AM_TMP_DIR); Path tmpDirPath = new Path(amTmpDir); Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir); fs.getFileSystem().mkdirs(launcherTmpDirPath); //launcher service launchService = new RoleLaunchService(this, providerService, fs, new Path(getGeneratedConfDir()), envVars, launcherTmpDirPath); runChildService(launchService); appState.noteAMLaunched(); //Give the provider restricted access to the state providerService.bind(appState); // launch the provider; this is expected to trigger a callback that // brings up the service launchProviderService(instanceDefinition, confDir); try { //now block waiting to be told to exit the process waitForAMCompletionSignal(); //shutdown time } finally { finish(); } return amExitCode; }
From source file:org.apache.oozie.action.hadoop.JHSCredentials.java
License:Apache License
/** * Create an MRClientProtocol to the JHS * Copied over from ClientCache in Hadoop. * @return the protocol that can be used to get a token with * @throws IOException/* ww w. j a v a 2 s. c o m*/ */ private MRClientProtocol instantiateHistoryProxy(final Configuration configuration, final ActionExecutor.Context context) throws IOException { final String serviceAddr = configuration.get(JHAdminConfig.MR_HISTORY_ADDRESS); if (StringUtils.isEmpty(serviceAddr)) { return null; } LOG.debug("Connecting to JHS at: " + serviceAddr); final YarnRPC rpc = YarnRPC.create(configuration); LOG.debug("Connected to JHS at: " + serviceAddr); UserGroupInformation currentUser = Services.get().get(UserGroupInformationService.class) .getProxyUser(context.getWorkflow().getUser()); return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() { @Override public MRClientProtocol run() { return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), configuration); } }); }