List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getYarnApplicationState
@Public @Stable public abstract YarnApplicationState getYarnApplicationState();
YarnApplicationState of the application. From source file:com.github.hdl.tensorflow.yarn.app.Client.java
License:Apache License
/** * Monitor the submitted application for completion. * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException/*w w w . j a v a2 s. co m*/ * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } ApplicationReport report = yarnClient.getApplicationReport(appId); // LOG.info("Got application report from ASM for" // + ", appId=" + appId.getId() // + ", clientToAMToken=" + report.getClientToAMToken() // + ", appDiagnostics=" + report.getDiagnostics() // + ", appMasterHost=" + report.getHost() // + ", appQueue=" + report.getQueue() // + ", appMasterRpcPort=" + report.getRpcPort() // + ", appStartTime=" + report.getStartTime() // + ", yarnAppState=" + report.getYarnApplicationState().toString() // + ", tfAppFinalState=" + report.getFinalApplicationStatus().toString() // + ", appTrackingUrl=" + report.getTrackingUrl() // + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus tfStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.RUNNING == state) { if (appRpc == null) { String hostname = report.getHost(); int port = report.getRpcPort(); LOG.info("application master rpc host: " + hostname + "; port: " + port); appRpc = new TFApplicationRpcClient(hostname, port).getRpc(); } if (appRpc != null && isEmptyString(clusterSpecJsonString)) { clusterSpecJsonString = appRpc.getClusterSpec(); LOG.info("cluster spec is " + clusterSpecJsonString); if (!isEmptyString(clusterSpecJsonString)) { TFClient tfClient = new TFClient(tfClientPy); if (isEnableTensorBoard) { Thread tensorBoardThread = new Thread() { @Override public void run() { tfClient.startTensorBoardClient(tensorboardEventDir); } }; tensorBoardThread.start(); LOG.info("Launching tensorboard ..."); try { Thread.sleep(3000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } if (tensorBoardThread.isAlive()) { LOG.info("the tensorboard launched successfully on the localhost:6006"); } else { LOG.info("the tensorboard launched failed"); } } tfClient.startTensorflowClient(clusterSpecJsonString); } } } if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == tfStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", tfAppFinalState=" + tfStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", tfAppFinalState=" + tfStatus.toString() + ". Breaking monitoring loop"); return false; } } }
From source file:com.github.sakserv.minicluster.simpleyarnapp.Client.java
License:Apache License
public void run(String[] args) throws Exception { final String command = args[0]; final int n = Integer.valueOf(args[1]); final Path jarPath = new Path(args[2]); final String resourceManagerAddress = args[3]; final String resourceManagerHostname = args[4]; final String resourceManagerSchedulerAddress = args[5]; final String resourceManagerResourceTrackerAddress = args[6]; // Create yarnClient YarnConfiguration conf = new YarnConfiguration(); conf.set("yarn.resourcemanager.address", resourceManagerAddress); conf.set("yarn.resourcemanager.hostname", resourceManagerHostname); conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress); conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);/* ww w .j ava2 s . c o m*/ yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " " + resourceManagerAddress + " " + resourceManagerHostname + " " + resourceManagerSchedulerAddress + " " + resourceManagerResourceTrackerAddress + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Setup jar for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); amContainer.setLocalResources(Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("simple-yarn-app"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); System.out.println("Submitting application " + appId); yarnClient.submitApplication(appContext); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(100); appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); } System.out.println("Application " + appId + " finished with" + " state " + appState + " at " + appReport.getFinishTime()); }
From source file:com.gpiskas.yarn.Client.java
License:Open Source License
private void run() throws Exception { conf = new YarnConfiguration(); // Create Yarn Client YarnClient client = YarnClient.createYarnClient(); client.init(conf);//from w w w . j a v a2s . c om client.start(); // Create Application YarnClientApplication app = client.createApplication(); // Create AM Container ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class); amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.gpiskas.yarn.AppMaster" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Set AM jar LocalResource jar = Records.newRecord(LocalResource.class); Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf); amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar)); // Set AM CLASSPATH Map<String, String> env = new HashMap<String, String>(); Utils.setUpEnv(env, conf); amCLC.setEnvironment(env); // Set AM resources Resource res = Records.newRecord(Resource.class); res.setMemory(256); res.setVirtualCores(1); // Create ApplicationSubmissionContext ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("YARNAPP"); appContext.setQueue("default"); appContext.setAMContainerSpec(amCLC); appContext.setResource(res); // Submit Application ApplicationId id = appContext.getApplicationId(); System.out.println("Client: Submitting " + id); client.submitApplication(appContext); ApplicationReport appReport = client.getApplicationReport(id); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(1000); appReport = client.getApplicationReport(id); appState = appReport.getYarnApplicationState(); } System.out.println("Client: Finished " + id + " with state " + appState); }
From source file:com.hazelcast.yarn.HazelcastYarnClient.java
License:Open Source License
private void init() throws Exception { YarnClient yarnClient = startYarnClient(this.conf); ApplicationId appId = submitApplication(yarnClient); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); appState = process(yarnClient, appId, appState); LOG.log(Level.INFO, "Application {0} is {1}.", new Object[] { appId, appState }); }
From source file:com.hazelcast.yarn.HazelcastYarnClient.java
License:Open Source License
private YarnApplicationState process(YarnClient yarnClient, ApplicationId appId, YarnApplicationState appState) throws InterruptedException, YarnException, IOException { ApplicationReport appReport; while (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED || appState == YarnApplicationState.ACCEPTED) { TimeUnit.SECONDS.sleep(1L); appReport = yarnClient.getApplicationReport(appId); if ((appState != YarnApplicationState.ACCEPTED) && (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED)) { LOG.log(Level.INFO, "Application {0} is ACCEPTED.", appId); }//from ww w. j a va 2 s .co m appState = appReport.getYarnApplicationState(); } return appState; }
From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java
License:Open Source License
/** * Method to launch the dml yarn app master and execute the given dml script * with the given configuration and jar file. * /* w ww . java 2 s.c om*/ * NOTE: on launching the yarn app master, we do not explicitly probe if we * are running on a yarn or MR1 cluster. In case of MR1, already the class * YarnConfiguration will not be found and raise a classnotfound. In case of any * exception we fall back to run CP directly in the client process. * * @return true if dml program successfully executed as yarn app master * @throws IOException */ protected boolean launchDMLYarnAppmaster() throws IOException, DMLScriptException { boolean ret = false; String hdfsWD = null; try { Timing time = new Timing(true); // load yarn configuration YarnConfiguration yconf = new YarnConfiguration(); // create yarn client YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yconf); yarnClient.start(); // create application and get the ApplicationID YarnClientApplication app = yarnClient.createApplication(); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); LOG.debug("Created application (applicationID: " + appId + ")"); // prepare hdfs working directory via ApplicationID // copy script, config, jar file to hdfs hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(_dmlConfig, appId); copyResourcesToHdfsWorkingDir(yconf, hdfsWD); //construct command line argument String command = constructAMCommand(_args, _dmlConfig); LOG.debug("Constructed application master command: \n" + command); // set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands(Collections.singletonList(command)); amContainer.setLocalResources(constructLocalResourceMap(yconf)); amContainer.setEnvironment(constructEnvionmentMap(yconf)); // Set up resource type requirements for ApplicationMaster int memHeap = _dmlConfig.getIntValue(DMLConfig.YARN_APPMASTERMEM); int memAlloc = (int) computeMemoryAllocation(memHeap); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memAlloc); capability.setVirtualCores(NUM_CORES); LOG.debug("Requested application resources: memory=" + memAlloc + ", vcores=" + NUM_CORES); // Finally, set-up ApplicationSubmissionContext for the application String qname = _dmlConfig.getTextValue(DMLConfig.YARN_APPQUEUE); appContext.setApplicationName(APPMASTER_NAME); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(qname); // queue LOG.debug("Configured application meta data: name=" + APPMASTER_NAME + ", queue=" + qname); // submit application (non-blocking) yarnClient.submitApplication(appContext); // Check application status periodically (and output web ui address) ApplicationReport appReport = yarnClient.getApplicationReport(appId); LOG.info("Application tracking-URL: " + appReport.getTrackingUrl()); YarnApplicationState appState = appReport.getYarnApplicationState(); YarnApplicationState oldState = appState; LOG.info("Application state: " + appState); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(APP_STATE_INTERVAL); //wait for 200ms appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); if (appState != oldState) { oldState = appState; LOG.info("Application state: " + appState); } } //check final status (failed or succeeded) FinalApplicationStatus finalState = appReport.getFinalApplicationStatus(); LOG.info("Application final status: " + finalState); //show application and total runtime double appRuntime = (double) (appReport.getFinishTime() - appReport.getStartTime()) / 1000; LOG.info("Application runtime: " + appRuntime + " sec."); LOG.info("Total runtime: " + String.format("%.3f", time.stop() / 1000) + " sec."); //raised script-level error in case of failed final status if (finalState != FinalApplicationStatus.SUCCEEDED) { //propagate script-level stop call message String stop_msg = readMessageToHDFSWorkingDir(_dmlConfig, yconf, appId); if (stop_msg != null) throw new DMLScriptException(stop_msg); //generic failure message throw new DMLRuntimeException( "DML yarn app master finished with final status: " + finalState + "."); } ret = true; } catch (DMLScriptException ex) { //rethrow DMLScriptException to propagate stop call throw ex; } catch (Exception ex) { LOG.error("Failed to run DML yarn app master.", ex); ret = false; } finally { //cleanup working directory if (hdfsWD != null) MapReduceTool.deleteFileIfExistOnHDFS(hdfsWD); } return ret; }
From source file:com.inforefiner.hdata.SubmitClient.java
License:Apache License
/** * Monitor the submitted application for completion. * Kill application if time expires.//from www .j a v a 2 s . co m * * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); // // List<ContainerReport> containerReports = yarnClient.getContainers(report.getCurrentApplicationAttemptId()); // for (ContainerReport containerReport : containerReports) { // String logUrl = containerReport.getLogUrl(); // LOG.info("Container log url = " + logUrl + ", host = "); // } // LOG.info("Got application report from ASM for" // + ", appId=" + appId.getId() // + ", clientToAMToken=" + report.getClientToAMToken() // + ", appDiagnostics=" + report.getDiagnostics() // + ", appMasterHost=" + report.getHost() // + ", appQueue=" + report.getQueue() // + ", appMasterRpcPort=" + report.getRpcPort() // + ", appStartTime=" + report.getStartTime() // + ", yarnAppState=" + report.getYarnApplicationState().toString() // + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() // + ", appTrackingUrl=" + report.getTrackingUrl() // + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
From source file:com.resa.yarn.Client.java
License:Open Source License
private void run() throws Exception { conf = new YarnConfiguration(); // Create Yarn Client YarnClient client = YarnClient.createYarnClient(); client.init(conf);//from w w w.j av a 2s . c om client.start(); // Create Application YarnClientApplication app = client.createApplication(); // Create AM Container ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class); amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.resa.yarn.AppMaster" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); // Set AM jar LocalResource jar = Records.newRecord(LocalResource.class); Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf); amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar)); // Set AM CLASSPATH Map<String, String> env = new HashMap<String, String>(); Utils.setUpEnv(env, conf); amCLC.setEnvironment(env); // Set AM resources Resource res = Records.newRecord(Resource.class); res.setMemory(256); res.setVirtualCores(1); // Create ApplicationSubmissionContext ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("YARNAPP"); appContext.setQueue("default"); appContext.setAMContainerSpec(amCLC); appContext.setResource(res); // Submit Application ApplicationId id = appContext.getApplicationId(); System.out.println("Client: Submitting " + id); client.submitApplication(appContext); ApplicationReport appReport = client.getApplicationReport(id); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(1000); appReport = client.getApplicationReport(id); appState = appReport.getYarnApplicationState(); } System.out.println("Client: Finished " + id + " with state " + appState); }
From source file:com.scistor.dshell.ScistorClient.java
License:Apache License
/** * Monitor the submitted application for completion. Kill application if * time expires./*from www. j ava 2 s . c o m*/ * * @param appId * Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 2 second. try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
From source file:com.sogou.dockeronyarn.client.DockerClient.java
License:Apache License
/** * Monitor the submitted application for completion. * Kill application if time expires. //ww w .j av a 2s. c om * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not shutdown." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }