List of usage examples for org.apache.hadoop.yarn.client.api YarnClient createYarnClient
@Public public static YarnClient createYarnClient()
From source file:UnmanagedAMLauncher.java
License:Apache License
public boolean init(String[] args) throws ParseException { Options opts = new Options(); opts.addOption("appname", true, "Application Name. Default value - UnmanagedAM"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); opts.addOption("cmd", true, "command to start unmanaged AM (required)"); opts.addOption("classpath", true, "additional classpath"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { printUsage(opts);//from w w w.ja v a 2 s . com throw new IllegalArgumentException("No args specified for client to initialize"); } if (cliParser.hasOption("help")) { printUsage(opts); return false; } appName = cliParser.getOptionValue("appname", "UnmanagedAM"); amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); amQueue = cliParser.getOptionValue("queue", "default"); classpath = cliParser.getOptionValue("classpath", null); amCmd = cliParser.getOptionValue("cmd"); if (amCmd == null) { printUsage(opts); throw new IllegalArgumentException("No cmd specified for application master"); } YarnConfiguration yarnConf = new YarnConfiguration(conf); rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); return true; }
From source file:alluxio.yarn.ApplicationMaster.java
License:Apache License
/** * Convenience constructor which uses the default Alluxio configuration. * * @param numWorkers the number of workers to launch * @param masterAddress the address at which to start the Alluxio master * @param resourcePath an hdfs path shared by all yarn nodes which can be used to share resources *//*from w w w . ja v a 2s . c o m*/ public ApplicationMaster(int numWorkers, String masterAddress, String resourcePath) { this(numWorkers, masterAddress, resourcePath, YarnClient.createYarnClient(), NMClient.createNMClient(), new AMRMClientAsyncFactory() { @Override public AMRMClientAsync<ContainerRequest> createAMRMClientAsync(int heartbeatMs, CallbackHandler handler) { return AMRMClientAsync.createAMRMClientAsync(heartbeatMs, handler); } }); }
From source file:alluxio.yarn.Client.java
License:Apache License
/** * Submits an application to the ResourceManager to run ApplicationMaster. * * The stable Yarn API provides a convenience method (YarnClient#createApplication) for creating * applications and setting up the application submission context. This was not available in the * alpha API./* ww w . j a v a 2 s.c o m*/ */ private void submitApplication() throws YarnException, IOException { // Initialize a YarnClient mYarnClient = YarnClient.createYarnClient(); mYarnClient.init(mYarnConf); mYarnClient.start(); // Create an application, get and check the information about the cluster YarnClientApplication app = mYarnClient.createApplication(); // Get a response of this application, containing information of the cluster GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // Check if the cluster has enough resource to launch the ApplicationMaster checkClusterResource(appResponse); // Check that there are enough hosts in the cluster to support the desired number of workers checkNodesAvailable(); // Set up the container launch context for the application master mAmContainer = Records.newRecord(ContainerLaunchContext.class); setupContainerLaunchContext(); // Finally, set-up ApplicationSubmissionContext for the application mAppContext = app.getApplicationSubmissionContext(); setupApplicationSubmissionContext(); // Submit the application to the applications manager. // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure mAppId = mAppContext.getApplicationId(); System.out.println("Submitting application of id " + mAppId + " to ResourceManager"); mYarnClient.submitApplication(mAppContext); monitorApplication(); }
From source file:alluxio.yarn.ClientTest.java
License:Apache License
@Before public void before() { // Mock Yarn client PowerMockito.mockStatic(YarnClient.class); mYarnClient = (YarnClient) Mockito.mock(YarnClient.class); Mockito.when(YarnClient.createYarnClient()).thenReturn(mYarnClient); }
From source file:azkaban.jobtype.HadoopJobUtils.java
License:Apache License
/** * <pre>/*from www. j a va2 s . co m*/ * Uses YarnClient to kill the job on HDFS. * Using JobClient only works partially: * If yarn container has started but spark job haven't, it will kill * If spark job has started, the cancel will hang until the spark job is complete * If the spark job is complete, it will return immediately, with a job not found on job tracker * </pre> * * @param applicationId * @throws IOException * @throws YarnException */ public static void killJobOnCluster(String applicationId, Logger log) throws YarnException, IOException { YarnConfiguration yarnConf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); yarnClient.start(); String[] split = applicationId.split("_"); ApplicationId aid = ApplicationId.newInstance(Long.parseLong(split[1]), Integer.parseInt(split[2])); log.info("start klling application: " + aid); yarnClient.killApplication(aid); log.info("successfully killed application: " + aid); }
From source file:cn.edu.buaa.act.petuumOnYarn.ApplicationMaster.java
License:Apache License
private boolean getAvaliableNodes() { List<NodeReport> clusterNodeReports; try {// w w w .j a v a2s.c o m YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport node : clusterNodeReports) { LOG.info("node infos:" + node.getHttpAddress()); } avaliableNodeList = new ArrayList<NodeReport>(); if (numNodes <= clusterNodeReports.size()) { for (NodeReport node : clusterNodeReports) { if (node.getCapability().getMemory() >= containerMemory && node.getCapability().getVirtualCores() >= containerVirtualCores) { avaliableNodeList.add(node); } } if (avaliableNodeList.size() >= numNodes) numTotalContainers = numNodes; else { LOG.error("Resource isn't enough"); return false; } } else { LOG.error("cluster nodes isn't enough"); return false; } } catch (Exception e) { LOG.error(e.getMessage()); LOG.error(e.getStackTrace()); return false; } return true; }
From source file:cn.edu.buaa.act.petuumOnYarn.Client.java
License:Apache License
Client(String appMasterMainClass, Configuration conf) {
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);//from www .ja va 2s.co m
opts = new Options();
opts.addOption("hdfs_path_prefix", true, "petuum dir path prefix on HDFS. default /petuum/");
opts.addOption("start_port", true, "Start port of each machine");
opts.addOption("num_nodes", true, "Required number of nodes");
opts.addOption("app_name", true, "Application Name. Default value - Petuum");
opts.addOption("priority", true, "Application Priority. Default 10");
opts.addOption("queue", true, "RM Queue in which this application is to be submitted.");
opts.addOption("master_memory", true,
"Amount of memory in MB to be requested to run the application master. Default 500");
opts.addOption("master_vcores", true,
"Amount of virtual cores to be requested to run the application master. Default 1");
opts.addOption("jar", true, "Jar file containing the application master");
opts.addOption("launch_script_path", true, "User's launch script path");
opts.addOption("shell_env", true, "Environment for script. Specified as env_key=env_val pairs");
opts.addOption("worker_priority", true, "Priority for the worker containers. Default 10");
opts.addOption("container_memory", true,
"Amount of memory in MB to be requested to run the worker. Default 1000");
opts.addOption("container_vcores", true,
"Amount of virtual cores to be requested to run the worker. Default 2");
opts.addOption("log_properties", true, "log4j.properties file");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application attempts."
+ " If the flag is true, running containers will not be killed when"
+ " application attempt fails and these containers will be retrieved by"
+ " the new application attempt ");
opts.addOption("attempt_failures_validity_interval", true,
"when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of "
+ "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed.");
opts.addOption("debug", false, "Dump out debug information");
opts.addOption("view_acls", true,
"Users and groups that allowed to " + "view the timeline entities in the given domain");
opts.addOption("modify_acls", true,
"Users and groups that allowed to " + "modify the timeline entities in the given domain");
opts.addOption("create", false,
"Flag to indicate whether to create the " + "domain specified with -domain.");
opts.addOption("help", false, "Print usage");
}
From source file:co.cask.cdap.common.security.YarnTokenUtils.java
License:Apache License
/** * Gets a Yarn delegation token and stores it in the given Credentials. * * @return the same Credentials instance as the one given in parameter. */// www.j a v a 2 s . co m public static Credentials obtainToken(YarnConfiguration configuration, Credentials credentials) { if (!UserGroupInformation.isSecurityEnabled()) { return credentials; } try { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(configuration); yarnClient.start(); try { Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName()); org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient .getRMDelegationToken(renewer); // TODO: The following logic should be replaced with call to ClientRMProxy.getRMDelegationTokenService after // CDAP-4825 is resolved List<String> services = new ArrayList<>(); if (HAUtil.isHAEnabled(configuration)) { // If HA is enabled, we need to enumerate all RM hosts // and add the corresponding service name to the token service // Copy the yarn conf since we need to modify it to get the RM addresses YarnConfiguration yarnConf = new YarnConfiguration(configuration); for (String rmId : HAUtil.getRMHAIds(configuration)) { yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); services.add(SecurityUtil.buildTokenService(address).toString()); } } else { services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString()); } Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken, (InetSocketAddress) null); token.setService(new Text(Joiner.on(',').join(services))); credentials.addToken(new Text(token.getService()), token); // OK to log, it won't log the credential, only information about the token. LOG.info("Added RM delegation token: {}", token); } finally { yarnClient.stop(); } return credentials; } catch (Exception e) { LOG.error("Failed to get secure token for Yarn.", e); throw Throwables.propagate(e); } }
From source file:co.cask.cdap.master.startup.YarnCheck.java
License:Apache License
@Override public void run() { int yarnConnectTimeout = cConf.getInt(Constants.Startup.YARN_CONNECT_TIMEOUT_SECONDS, 60); LOG.info("Checking YARN availability -- may take up to {} seconds.", yarnConnectTimeout); final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(hConf);/*from www . j a v a 2 s . com*/ List<NodeReport> nodeReports; // if yarn is not up, yarnClient.start() will hang. ExecutorService executorService = Executors .newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("startup-checker").build()); try { Future<List<NodeReport>> result = executorService.submit(new Callable<List<NodeReport>>() { @Override public List<NodeReport> call() throws Exception { yarnClient.start(); return yarnClient.getNodeReports(); } }); nodeReports = result.get(yarnConnectTimeout, TimeUnit.SECONDS); LOG.info(" YARN availability successfully verified."); } catch (Exception e) { throw new RuntimeException("Unable to get status of YARN nodemanagers. " + "Please check that YARN is running " + "and that the correct Hadoop configuration (core-site.xml, yarn-site.xml) and libraries " + "are included in the CDAP master classpath.", e); } finally { try { yarnClient.stop(); } catch (Exception e) { LOG.warn("Error stopping yarn client.", e); } finally { executorService.shutdown(); } } checkResources(nodeReports); }
From source file:co.cask.cdap.operations.yarn.AbstractYarnStats.java
License:Apache License
protected YarnClient createYARNClient() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); return yarnClient; }