List of usage examples for org.apache.hadoop.yarn.api.protocolrecords GetNewApplicationResponse getApplicationId
@Public @Stable public abstract ApplicationId getApplicationId();
ApplicationId allocated by the ResourceManager. From source file:com.cloudera.kitten.client.service.YarnClientServiceImpl.java
License:Open Source License
@Override protected void startUp() { this.applicationsManager = applicationsManagerFactory.connect(); GetNewApplicationResponse newApp = getNewApplication(); this.applicationId = newApp.getApplicationId(); ContainerLaunchContextFactory clcFactory = new ContainerLaunchContextFactory( newApp.getMinimumResourceCapability(), newApp.getMaximumResourceCapability()); LOG.info("Setting up application submission context for the application master"); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationId(applicationId); appContext.setApplicationName(parameters.getApplicationName()); // Setup the container for the application master. ContainerLaunchParameters appMasterParams = parameters.getApplicationMasterParameters(applicationId); ContainerLaunchContext clc = clcFactory.create(appMasterParams); appContext.setAMContainerSpec(clc);/*from ww w . java2 s .co m*/ appContext.setUser(appMasterParams.getUser()); appContext.setQueue(parameters.getQueue()); appContext.setPriority(ContainerLaunchContextFactory.createPriority(appMasterParams.getPriority())); submitApplication(appContext); stopwatch.start(); }
From source file:com.continuuity.weave.internal.yarn.Hadoop20YarnAppClient.java
License:Apache License
@Override public ProcessLauncher<ApplicationId> createLauncher(WeaveSpecification weaveSpec) throws Exception { // Request for new application final GetNewApplicationResponse response = yarnClient.getNewApplication(); final ApplicationId appId = response.getApplicationId(); // Setup the context for application submission final ApplicationSubmissionContext appSubmissionContext = Records .newRecord(ApplicationSubmissionContext.class); appSubmissionContext.setApplicationId(appId); appSubmissionContext.setApplicationName(weaveSpec.getName()); appSubmissionContext.setUser(user);/*from w w w. jav a 2 s .c o m*/ ApplicationSubmitter submitter = new ApplicationSubmitter() { @Override public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext, Resource capability) { ContainerLaunchContext context = launchContext.getLaunchContext(); addRMToken(context); context.setUser(appSubmissionContext.getUser()); context.setResource(adjustMemory(response, capability)); appSubmissionContext.setAMContainerSpec(context); try { yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(yarnClient, appId); } catch (YarnRemoteException e) { LOG.error("Failed to submit application {}", appId, e); throw Throwables.propagate(e); } } }; return new ApplicationMasterProcessLauncher(appId, submitter); }
From source file:com.continuuity.weave.internal.yarn.Hadoop21YarnAppClient.java
License:Apache License
@Override public ProcessLauncher<ApplicationId> createLauncher(WeaveSpecification weaveSpec) throws Exception { // Request for new application YarnClientApplication application = yarnClient.createApplication(); final GetNewApplicationResponse response = application.getNewApplicationResponse(); final ApplicationId appId = response.getApplicationId(); // Setup the context for application submission final ApplicationSubmissionContext appSubmissionContext = application.getApplicationSubmissionContext(); appSubmissionContext.setApplicationId(appId); appSubmissionContext.setApplicationName(weaveSpec.getName()); ApplicationSubmitter submitter = new ApplicationSubmitter() { @Override//from w w w .j a v a 2 s.co m public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context, Resource capability) { ContainerLaunchContext launchContext = context.getLaunchContext(); addRMToken(launchContext); appSubmissionContext.setAMContainerSpec(launchContext); appSubmissionContext.setResource(adjustMemory(response, capability)); appSubmissionContext.setMaxAppAttempts(2); try { yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(yarnClient, appId); } catch (Exception e) { LOG.error("Failed to submit application {}", appId, e); throw Throwables.propagate(e); } } }; return new ApplicationMasterProcessLauncher(appId, submitter); }
From source file:com.continuuity.weave.yarn.YarnWeavePreparer.java
License:Open Source License
@Override public WeaveController start() { // TODO: Unify this with {@link ProcessLauncher} try {/*from w w w . ja va2 s. c o m*/ GetNewApplicationResponse response = yarnClient.getNewApplication(); ApplicationId applicationId = response.getApplicationId(); ApplicationSubmissionContext appSubmissionContext = Records .newRecord(ApplicationSubmissionContext.class); appSubmissionContext.setApplicationId(applicationId); appSubmissionContext.setApplicationName(weaveSpec.getName()); Map<String, LocalResource> localResources = Maps.newHashMap(); Multimap<String, LocalFile> transformedLocalFiles = HashMultimap.create(); createAppMasterJar(createBundler(), localResources); createContainerJar(createBundler(), localResources); populateRunnableResources(weaveSpec, transformedLocalFiles); saveWeaveSpec(weaveSpec, transformedLocalFiles, localResources); saveLogback(localResources); saveLauncher(localResources); saveKafka(localResources); saveArguments(arguments, runnableArgs, localResources); saveLocalFiles(localResources, ImmutableSet.of("weaveSpec.json", "logback-template.xml", "container.jar", "launcher.jar", "arguments.json")); ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(localResources); // java -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory // com.continuuity.weave.internal.WeaveLauncher // appMaster.jar // com.continuuity.weave.internal.appmaster.ApplicationMasterMain // false containerLaunchContext.setCommands(ImmutableList.of("java", "-cp", "launcher.jar:$HADOOP_CONF_DIR", "-Xmx" + APP_MASTER_MEMORY_MB + "m", WeaveLauncher.class.getName(), "appMaster.jar", ApplicationMasterMain.class.getName(), Boolean.FALSE.toString(), " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout", " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); containerLaunchContext.setEnvironment(ImmutableMap.<String, String>builder() .put(EnvKeys.WEAVE_APP_ID, Integer.toString(applicationId.getId())) .put(EnvKeys.WEAVE_APP_ID_CLUSTER_TIME, Long.toString(applicationId.getClusterTimestamp())) .put(EnvKeys.WEAVE_APP_DIR, getAppLocation().toURI().toASCIIString()) .put(EnvKeys.WEAVE_ZK_CONNECT, zkClient.getConnectString()) .put(EnvKeys.WEAVE_RUN_ID, runId.getId()).build()); Resource capability = Records.newRecord(Resource.class); capability.setMemory(APP_MASTER_MEMORY_MB); containerLaunchContext.setResource(capability); appSubmissionContext.setAMContainerSpec(containerLaunchContext); yarnClient.submitApplication(appSubmissionContext); return createController(applicationId, runId, logHandlers); } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:com.topekalabs.bigmachine.client.BGMClient.java
public void launch() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration()); yarnClient.start();/*from ww w. j av a2 s .c o m*/ YarnClientApplication app = null; try { app = yarnClient.createApplication(); } catch (YarnException | IOException ex) { throw new RuntimeException(ex); } GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); logger.debug("Application ID {}", appResponse.getApplicationId()); logger.debug("Maximum available resources: {}", appResponse.getMaximumResourceCapability()); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); ContainerLaunchContext appMasterContainer = Records.newRecord(ContainerLaunchContext.class); }
From source file:com.toy.Client.java
License:Apache License
/** * Start a new Application Master and deploy the web application on 2 Tomcat containers * * @throws Exception//from ww w . ja v a2 s .c om */ void start() throws Exception { //Check tomcat dir final File tomcatHomeDir = new File(toyConfig.tomcat); final File tomcatLibraries = new File(tomcatHomeDir, "lib"); final File tomcatBinaries = new File(tomcatHomeDir, "bin"); Preconditions.checkState(tomcatLibraries.isDirectory(), tomcatLibraries.getAbsolutePath() + " does not exist"); //Check war file final File warFile = new File(toyConfig.war); Preconditions.checkState(warFile.isFile(), warFile.getAbsolutePath() + " does not exist"); yarn = YarnClient.createYarnClient(); yarn.init(configuration); yarn.start(); YarnClientApplication yarnApplication = yarn.createApplication(); GetNewApplicationResponse newApplication = yarnApplication.getNewApplicationResponse(); appId = newApplication.getApplicationId(); ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext(); appContext.setApplicationName("Tomcat : " + tomcatHomeDir.getName() + "\n War : " + warFile.getName()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // Register required libraries Map<String, LocalResource> localResources = new HashMap<>(); FileSystem fs = FileSystem.get(configuration); uploadDepAndRegister(localResources, appId, fs, "lib-ext/curator-client-2.3.0.jar"); uploadDepAndRegister(localResources, appId, fs, "lib-ext/curator-framework-2.3.0.jar"); uploadDepAndRegister(localResources, appId, fs, "lib-ext/curator-recipes-2.3.0.jar"); // Register application master jar registerLocalResource(localResources, appId, fs, new Path(appMasterJar)); // Register the WAR that will be deployed on Tomcat registerLocalResource(localResources, appId, fs, new Path(warFile.getAbsolutePath())); // Register Tomcat libraries for (File lib : tomcatLibraries.listFiles()) { registerLocalResource(localResources, appId, fs, new Path(lib.getAbsolutePath())); } File juli = new File(tomcatBinaries, "tomcat-juli.jar"); if (juli.exists()) { registerLocalResource(localResources, appId, fs, new Path(juli.getAbsolutePath())); } amContainer.setLocalResources(localResources); // Setup master environment Map<String, String> env = new HashMap<>(); final String TOMCAT_LIBS = fs.getHomeDirectory() + "/" + Constants.TOY_PREFIX + appId.toString(); env.put(Constants.TOMCAT_LIBS, TOMCAT_LIBS); if (toyConfig.zookeeper != null) { env.put(Constants.ZOOKEEPER_QUORUM, toyConfig.zookeeper); } else { env.put(Constants.ZOOKEEPER_QUORUM, NetUtils.getHostname()); } // 1. Compute classpath StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$()) .append(File.pathSeparatorChar).append("./*"); for (String c : configuration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (configuration.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); env.put(Constants.WAR, warFile.getName()); // For unit test with YarnMiniCluster env.put(YarnConfiguration.RM_SCHEDULER_ADDRESS, configuration.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); amContainer.setEnvironment(env); // 1.2 Set constraint for the app master Resource capability = Records.newRecord(Resource.class); capability.setMemory(32); appContext.setResource(capability); // 2. Compute app master cmd line Vector<CharSequence> vargs = new Vector<>(10); // Set java executable command vargs.add(ApplicationConstants.Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx32m"); // Set class name vargs.add(TOYMaster.class.getCanonicalName()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<>(); commands.add(command.toString()); amContainer.setCommands(commands); appContext.setAMContainerSpec(amContainer); // 3. Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = configuration.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new Exception("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final org.apache.hadoop.security.token.Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (org.apache.hadoop.security.token.Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setQueue("default"); LOG.info("Submitting TOY application {} to ASM", appId.toString()); yarn.submitApplication(appContext); // Monitor the application and exit if it is RUNNING monitorApplication(appId); }
From source file:com.yahoo.storm.yarn.StormOnYarn.java
License:Open Source License
private void launchApp(String appName, String queue, int amMB, String storm_zip_location) throws Exception { LOG.debug("StormOnYarn:launchApp() ..."); YarnClientApplication client_app = _yarn.createApplication(); GetNewApplicationResponse app = client_app.getNewApplicationResponse(); _appId = app.getApplicationId(); LOG.debug("_appId:" + _appId); if (amMB > app.getMaximumResourceCapability().getMemory()) { //TODO need some sanity checks amMB = app.getMaximumResourceCapability().getMemory(); }//from w w w. ja va2 s .c o m ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationId(app.getApplicationId()); appContext.setApplicationName(appName); appContext.setQueue(queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the // local resources LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path String appMasterJar = findContainingJar(MasterServer.class); FileSystem fs = FileSystem.get(_hadoopConf); Path src = new Path(appMasterJar); String appHome = Util.getApplicationHomeForId(_appId.toString()); Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar"); fs.copyFromLocalFile(false, true, src, dst); localResources.put("AppMaster.jar", Util.newYarnAppResource(fs, dst)); String stormVersion = Util.getStormVersion(); Path zip; if (storm_zip_location != null) { zip = new Path(storm_zip_location); } else { zip = new Path("/lib/storm/" + stormVersion + "/storm.zip"); } _stormConf.put("storm.zip.path", zip.makeQualified(fs).toUri().getPath()); LocalResourceVisibility visibility = LocalResourceVisibility.PUBLIC; _stormConf.put("storm.zip.visibility", "PUBLIC"); if (!Util.isPublic(fs, zip)) { visibility = LocalResourceVisibility.APPLICATION; _stormConf.put("storm.zip.visibility", "APPLICATION"); } localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, visibility)); Path confDst = Util.createConfigurationFileInFs(fs, appHome, _stormConf, _hadoopConf); // establish a symbolic link to conf directory localResources.put("conf", Util.newYarnAppResource(fs, confDst)); // Setup security tokens Path[] paths = new Path[3]; paths[0] = dst; paths[1] = zip; paths[2] = confDst; Credentials credentials = new Credentials(); TokenCache.obtainTokensForNamenodes(credentials, paths, _hadoopConf); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); //security tokens for HDFS distributed cache amContainer.setTokens(securityTokens); // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the env variables to be setup in the env where the application master // will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // add the runtime classpath needed for tests to work Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./conf"); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar"); //Make sure that AppMaster has access to all YARN JARs List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath"); ProcessBuilder pb = new ProcessBuilder(yarn_classpath_cmd).redirectError(Redirect.INHERIT); LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]"); pb.environment().putAll(System.getenv()); Process proc = pb.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), "UTF-8")); String line = ""; String yarn_class_path = (String) _stormConf.get("storm.yarn.yarn_classpath"); if (yarn_class_path == null) { StringBuilder yarn_class_path_builder = new StringBuilder(); while ((line = reader.readLine()) != null) { yarn_class_path_builder.append(line); } yarn_class_path = yarn_class_path_builder.toString(); } LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]"); proc.waitFor(); reader.close(); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path); String stormHomeInZip = Util.getStormHomeInZip(fs, zip, stormVersion); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/*"); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/lib/*"); String java_home = (String) _stormConf.get("storm.yarn.java_home"); if (java_home == null) java_home = System.getenv("JAVA_HOME"); if (java_home != null && !java_home.isEmpty()) env.put("JAVA_HOME", java_home); LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]"); env.put("appJar", appMasterJar); env.put("appName", appName); env.put("appId", new Integer(_appId.getId()).toString()); env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<String> vargs = new Vector<String>(); if (java_home != null && !java_home.isEmpty()) vargs.add(env.get("JAVA_HOME") + "/bin/java"); else vargs.add("java"); vargs.add("-Dstorm.home=./storm/" + stormHomeInZip + "/"); vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log"); //vargs.add("-verbose:class"); vargs.add("com.yahoo.storm.yarn.MasterServer"); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); // Set java executable command LOG.info("Setting up app master command:" + vargs); amContainer.setCommands(vargs); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMB); appContext.setResource(capability); appContext.setAMContainerSpec(amContainer); _yarn.submitApplication(appContext); }
From source file:com.yss.yarn.launch.APPLaunch.java
License:Open Source License
private void launchApp(String appName, String queue, int amMB, String appMasterJar, String classPath, String lanchMainClass, Map<String, String> runenv) throws Exception { YarnClientApplication client_app = _yarn.createApplication(); GetNewApplicationResponse app = client_app.getNewApplicationResponse(); _appId = app.getApplicationId(); LOG.debug("_appId:" + _appId); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationId(app.getApplicationId()); appContext.setApplicationName(appName); appContext.setQueue(queue);/* ww w. ja va2 s. c om*/ ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); LOG.info("load " + appMasterJar); FileSystem fs = FileSystem.get(_hadoopConf); Path src = new Path(appMasterJar); String appHome = YarnUtil.getApplicationHomeForId(_appId.toString()); Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar"); fs.copyFromLocalFile(false, true, src, dst); localResources.put("AppMaster.jar", YarnUtil.newYarnAppResource(fs, dst)); amContainer.setLocalResources(localResources); LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar"); List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath"); LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]"); String yarn_class_path = classPath; LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]"); Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path); String java_home = System.getenv("JAVA_HOME"); if ((java_home != null) && !java_home.isEmpty()) { env.put("JAVA_HOME", java_home); } LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]"); env.put("appJar", appMasterJar); env.put("appName", appName); env.put("appId", _appId.toString()); env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); env.putAll(runenv); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<String> vargs = new Vector<String>(); if ((java_home != null) && !java_home.isEmpty()) { vargs.add(env.get("JAVA_HOME") + "/bin/java"); } else { vargs.add("java"); } vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log"); vargs.add(lanchMainClass); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Set java executable command LOG.info("Setting up app master command:" + vargs); amContainer.setCommands(vargs); Resource capability = Records.newRecord(Resource.class); appContext.setResource(capability); appContext.setAMContainerSpec(amContainer); appContext.setApplicationName(appName); _yarn.submitApplication(appContext); }
From source file:edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnApplication.java
License:Apache License
private static ApplicationId getNewApplicationId(YarnClientRMConnection crmc) throws YarnRemoteException { GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse response = crmc.getClientRMProtocol().getNewApplication(request); return response.getApplicationId(); }
From source file:io.amient.yarn1.YarnClient.java
License:Open Source License
/** * This method should be called by the implementing application static main * method. It does all the work around creating a yarn application and * submitting the request to the yarn resource manager. The class given in * the appClass argument will be run inside the yarn-allocated master * container.//from w w w . j a v a 2 s. c o m */ public static void submitApplicationMaster(Properties appConfig, Class<? extends YarnMaster> masterClass, String[] args, Boolean awaitCompletion) throws Exception { log.info("Yarn1 App Configuration:"); for (Object param : appConfig.keySet()) { log.info(param.toString() + " = " + appConfig.get(param).toString()); } String yarnConfigPath = appConfig.getProperty("yarn1.site", "/etc/hadoop"); String masterClassName = masterClass.getName(); appConfig.setProperty("yarn1.master.class", masterClassName); String applicationName = appConfig.getProperty("yarn1.application.name", masterClassName); log.info("--------------------------------------------------------------"); if (Boolean.valueOf(appConfig.getProperty("yarn1.local.mode", "false"))) { YarnMaster.run(appConfig, args); return; } int masterPriority = Integer.valueOf( appConfig.getProperty("yarn1.master.priority", String.valueOf(YarnMaster.DEFAULT_MASTER_PRIORITY))); int masterMemoryMb = Integer.valueOf(appConfig.getProperty("yarn1.master.memory.mb", String.valueOf(YarnMaster.DEFAULT_MASTER_MEMORY_MB))); int masterNumCores = Integer.valueOf( appConfig.getProperty("yarn1.master.num.cores", String.valueOf(YarnMaster.DEFAULT_MASTER_CORES))); String queue = appConfig.getProperty("yarn1.queue"); Configuration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new FileInputStream(yarnConfigPath + "/core-site.xml")); yarnConfig.addResource(new FileInputStream(yarnConfigPath + "/hdfs-site.xml")); yarnConfig.addResource(new FileInputStream(yarnConfigPath + "/yarn-site.xml")); for (Map.Entry<Object, Object> entry : appConfig.entrySet()) { yarnConfig.set(entry.getKey().toString(), entry.getValue().toString()); } final org.apache.hadoop.yarn.client.api.YarnClient yarnClient = org.apache.hadoop.yarn.client.api.YarnClient .createYarnClient(); yarnClient.init(yarnConfig); yarnClient.start(); for (NodeReport report : yarnClient.getNodeReports(NodeState.RUNNING)) { log.debug("Node report:" + report.getNodeId() + " @ " + report.getHttpAddress() + " | " + report.getCapability()); } log.info("Submitting application master class " + masterClassName); YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); final ApplicationId appId = appResponse.getApplicationId(); if (appId == null) { System.exit(111); } else { appConfig.setProperty("am.timestamp", String.valueOf(appId.getClusterTimestamp())); appConfig.setProperty("am.id", String.valueOf(appId.getId())); } YarnClient.distributeResources(yarnConfig, appConfig, applicationName); String masterJvmArgs = appConfig.getProperty("yarn1.master.jvm.args", ""); YarnContainerContext masterContainer = new YarnContainerContext(yarnConfig, appConfig, masterJvmArgs, masterPriority, masterMemoryMb, masterNumCores, applicationName, YarnMaster.class, args); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(masterClassName); appContext.setResource(masterContainer.capability); appContext.setPriority(masterContainer.priority); appContext.setQueue(queue); appContext.setApplicationType(appConfig.getProperty("yarn1.application.type", "YARN")); appContext.setAMContainerSpec(masterContainer.createContainerLaunchContext()); log.info("Master container spec: " + masterContainer.capability); yarnClient.submitApplication(appContext); ApplicationReport report = yarnClient.getApplicationReport(appId); log.info("Tracking URL: " + report.getTrackingUrl()); if (awaitCompletion) { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { if (!yarnClient.isInState(Service.STATE.STOPPED)) { log.info("Killing yarn application in shutdown hook"); try { yarnClient.killApplication(appId); } catch (Throwable e) { log.error("Failed to kill yarn application - please check YARN Resource Manager", e); } } } }); float lastProgress = -0.0f; while (true) { try { Thread.sleep(10000); report = yarnClient.getApplicationReport(appId); if (lastProgress != report.getProgress()) { lastProgress = report.getProgress(); log.info(report.getApplicationId() + " " + (report.getProgress() * 100.00) + "% " + (System.currentTimeMillis() - report.getStartTime()) + "(ms) " + report.getDiagnostics()); } if (!report.getFinalApplicationStatus().equals(FinalApplicationStatus.UNDEFINED)) { log.info(report.getApplicationId() + " " + report.getFinalApplicationStatus()); log.info("Tracking url: " + report.getTrackingUrl()); log.info("Finish time: " + ((System.currentTimeMillis() - report.getStartTime()) / 1000) + "(s)"); break; } } catch (Throwable e) { log.error("Master Heart Beat Error - terminating", e); yarnClient.killApplication(appId); Thread.sleep(2000); } } yarnClient.stop(); if (!report.getFinalApplicationStatus().equals(FinalApplicationStatus.SUCCEEDED)) { System.exit(112); } } yarnClient.stop(); }