List of usage examples for org.apache.hadoop.yarn.client.api YarnClient createYarnClient
@Public public static YarnClient createYarnClient()
From source file:uk.gov.gchq.gaffer.slider.util.AppConfigGenerator.java
License:Apache License
private AvailableResources getYarnResources() throws IOException, YarnException { final Configuration config = new Configuration(); final YarnClient yarn = YarnClient.createYarnClient(); yarn.init(config);//from ww w . j av a2 s.c om yarn.start(); // Query YARN to find out the largest container it is capable of scheduling final YarnClientApplication app = yarn.createApplication(); final Resource resources = app.getNewApplicationResponse().getMaximumResourceCapability(); // Also find out how many nodes there are in the cluster by asking for the number of registered Node Managers final YarnClusterMetrics metrics = yarn.getYarnClusterMetrics(); yarn.close(); return new AvailableResources(resources.getVirtualCores(), resources.getMemory(), metrics.getNumNodeManagers()); }
From source file:x10.x10rt.yarn.Client.java
License:Open Source License
public Client(String[] args) { this.conf = new YarnConfiguration(); //this.conf.set(YarnConfiguration.RM_HOSTNAME, args[0]); //this.conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 1000); //this.conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000); //this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://"+args[0]); this.appMasterMainClass = ApplicationMaster.class.getName(); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(this.conf); this.args = args; // find the original classpath argument from the x10 script String prefix = "-Djava.class.path="; for (int classPathArg = 0; classPathArg < args.length; classPathArg++) { if (args[classPathArg].startsWith(prefix)) { classPath = args[classPathArg].substring(prefix.length()); this.classPathArg = classPathArg; break; }/*from w w w . j a v a2 s . com*/ } // find the first non-jvm argument, which is the main class name int mainClassArg = 0; for (; mainClassArg < args.length; mainClassArg++) { if (args[mainClassArg].charAt(0) != '-') break; } this.mainClassArg = mainClassArg; int lastslash = args[mainClassArg].lastIndexOf('/'); if (lastslash == -1) this.appName = args[mainClassArg]; else this.appName = args[mainClassArg].substring(lastslash + 1); }
From source file:yarnkit.client.YarnClientService.java
License:Apache License
@Nonnull private YarnClient connect() { YarnClient client = YarnClient.createYarnClient(); client.init(conf); client.start(); return client; }
From source file:yrun.commands.Ykill.java
License:Apache License
public static void main(String[] args) throws IOException { YarnClient yarnClient = YarnClient.createYarnClient(); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnClient.init(yarnConfiguration);/*from w ww. j a v a2s .com*/ yarnClient.start(); boolean debug = false; try { Splitter splitter = Splitter.on('_'); for (String arg : args) { List<String> list = toList(splitter.split(arg)); if (list.size() != 3) { System.err.println("Application Id " + arg + " is not a valid application id."); } else { String prefix = list.get(0); if (!prefix.equals("application")) { System.err.println("Application Id " + arg + " is not a valid application id."); } else { try { long clusterTimestamp = Long.parseLong(list.get(1)); int id = Integer.parseInt(list.get(2)); ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp, id); yarnClient.killApplication(applicationId); System.out.println("Killed\t" + arg + ""); } catch (Exception e) { if (debug) { e.printStackTrace(); } System.err.println("Error while trying to kill " + arg + "."); } } } } } finally { yarnClient.stop(); yarnClient.close(); } }
From source file:yrun.commands.Yps.java
License:Apache License
public static void main(String[] args) throws YarnException, IOException { YarnClient yarnClient = YarnClient.createYarnClient(); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnClient.init(yarnConfiguration);// www . j a va 2s .com yarnClient.start(); try { List<ApplicationReport> applications = yarnClient.getApplications(); for (ApplicationReport applicationReport : applications) { ApplicationId applicationId = applicationReport.getApplicationId(); String user = applicationReport.getUser(); String queue = applicationReport.getQueue(); String name = applicationReport.getName(); YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); float progress = applicationReport.getProgress(); System.out.printf("%s\t%s\t%s\t%s\t%s\t%f%n", toString(applicationId), user, queue, name, yarnApplicationState.name(), progress); } } finally { yarnClient.stop(); yarnClient.close(); } }
From source file:yrun.YarnRunner.java
License:Apache License
public void execute() throws IOException, YarnException, InterruptedException { LOG.info("Using application path [" + _installPath + "]"); Path jarPath = installThisJar(_installPath, _appJarFile); LOG.info("Driver installed [" + jarPath + "]"); List<Path> installedArchivePathList = install(_installPath, _archivePathList); for (Path p : installedArchivePathList) { LOG.info("Archive installed [" + p + "]"); }//from ww w . j a v a 2s.co m YarnRunnerArgs yarnRunnerArgs = new YarnRunnerArgs(); yarnRunnerArgs.setCommand(_command); Path argsPath = installThisArgs(_installPath, yarnRunnerArgs); final YarnClient client = YarnClient.createYarnClient(); _configuration.setInt("yarn.nodemanager.delete.debug-delay-sec", (int) TimeUnit.HOURS.toSeconds(1)); client.init(_configuration); client.start(); YarnClientApplication app = client.createApplication(); ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv, _appJarFile); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); { LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); localResources.put(jarPath.getName(), appMasterJar); } { LocalResource appMasterArgs = Records.newRecord(LocalResource.class); setupAppMasterArgs(argsPath, appMasterArgs); localResources.put(MASTER_JSON, appMasterArgs); } List<String> vargs = new ArrayList<String>(); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-Xmx256m"); vargs.add("-Djava.net.preferIPv4Stack=true"); vargs.add(YarnRunnerApplicationMaster.class.getName()); String strCommand = "(echo ENV && set && echo CURRENT_DIR_LISTING && ls -la && echo PWD && pwd && (" + StringUtils.join(" ", vargs) + "))"; strCommand += " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"; strCommand += " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; LOG.debug("Application Master command [" + strCommand + "]"); amContainer.setCommands(Collections.singletonList(strCommand)); amContainer.setLocalResources(localResources); amContainer.setEnvironment(appMasterEnv); Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(_yarnName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); if (_queue != null) { appContext.setQueue(_queue); } appContext.setApplicationType("yrun"); ApplicationId appId = appContext.getApplicationId(); AtomicBoolean shutdown = new AtomicBoolean(); if (!_isDaemon) { addShutdownHook(client, appId, shutdown); } LOG.info("Submitting application with id [" + appId + "]"); client.submitApplication(appContext); ApplicationReport report; YarnApplicationState state; do { report = client.getApplicationReport(appId); state = report.getYarnApplicationState(); if (state == YarnApplicationState.RUNNING) { if (_isDaemon) { LOG.info("Application is running. This is a daemon application driver program exiting."); return; } } Thread.sleep(100); } while (isNoLongerRunning(state)); shutdown.set(true); LOG.info("Application has finished with state [" + state + "]"); }