List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getYarnApplicationState
@Public @Stable public abstract YarnApplicationState getYarnApplicationState();
YarnApplicationState of the application. From source file:org.apache.hoya.yarn.client.HoyaClient.java
License:Apache License
/** * Test for a cluster existing probe for a cluster of the given name existing * in the filesystem. If the live param is set, it must be a live cluster * @return exit code//from w ww. java 2s .c o m */ @VisibleForTesting public int actionExists(String name, boolean live) throws YarnException, IOException { verifyManagerSet(); HoyaUtils.validateClusterName(name); log.debug("actionExists({}, {})", name, live); //initial probe for a cluster in the filesystem Path clusterDirectory = hoyaFileSystem.buildHoyaClusterDirPath(name); if (!hoyaFileSystem.getFileSystem().exists(clusterDirectory)) { throw unknownClusterException(name); } //test for liveness if desired if (live) { ApplicationReport instance = findInstance(name); if (instance == null) { log.info("cluster {} not running", name); return EXIT_FALSE; } else { // the app exists, but it may be in a terminated state HoyaUtils.OnDemandReportStringifier report = new HoyaUtils.OnDemandReportStringifier(instance); YarnApplicationState state = instance.getYarnApplicationState(); if (state.ordinal() >= YarnApplicationState.FINISHED.ordinal()) { //cluster in the list of apps but not running log.info("Cluster {} found but is in state {}", name, state); log.debug("State {}", report); return EXIT_FALSE; } log.info("Cluster {} is running:\n{}", name, report); } } else { log.info("Cluster {} exists but is not running", name); } return EXIT_SUCCESS; }
From source file:org.apache.hoya.yarn.client.HoyaClient.java
License:Apache License
/** * Freeze the cluster/*from www .j av a 2 s. c o m*/ * * @param clustername cluster name * @param freezeArgs arguments to the freeze * @return EXIT_SUCCESS if the cluster was not running by the end of the operation */ public int actionFreeze(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException { verifyManagerSet(); HoyaUtils.validateClusterName(clustername); int waittime = freezeArgs.getWaittime(); String text = freezeArgs.message; boolean forcekill = freezeArgs.force; log.debug("actionFreeze({}, reason={}, wait={}, force={})", clustername, text, waittime, forcekill); //is this actually a known cluster? hoyaFileSystem.locateInstanceDefinition(clustername); ApplicationReport app = findInstance(clustername); if (app == null) { // exit early log.info("Cluster {} not running", clustername); // not an error to freeze a frozen cluster return EXIT_SUCCESS; } log.debug("App to freeze was found: {}:\n{}", clustername, new HoyaUtils.OnDemandReportStringifier(app)); if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) { log.info("Cluster {} is a terminated state {}", clustername, app.getYarnApplicationState()); return EXIT_SUCCESS; } LaunchedApplication application = new LaunchedApplication(yarnClient, app); applicationId = application.getApplicationId(); if (forcekill) { //escalating to forced kill application.kill("Forced freeze of " + clustername + ": " + text); } else { try { HoyaClusterProtocol appMaster = connect(app); Messages.StopClusterRequestProto r = Messages.StopClusterRequestProto.newBuilder().setMessage(text) .build(); appMaster.stopCluster(r); log.debug("Cluster stop command issued"); } catch (YarnException e) { log.warn("Exception while trying to terminate {}: {}", clustername, e); return EXIT_FALSE; } catch (IOException e) { log.warn("Exception while trying to terminate {}: {}", clustername, e); return EXIT_FALSE; } } //wait for completion. We don't currently return an exception during this process //as the stop operation has been issued, this is just YARN. try { if (waittime > 0) { ApplicationReport applicationReport = application.monitorAppToState(YarnApplicationState.FINISHED, new Duration(waittime * 1000)); if (applicationReport == null) { log.info("application did not shut down in time"); return EXIT_FALSE; } } } catch (YarnException e) { log.warn("Exception while waiting for the cluster {} to shut down: {}", clustername, e); } catch (IOException e) { log.warn("Exception while waiting for the cluster {} to shut down: {}", clustername, e); } return EXIT_SUCCESS; }
From source file:org.apache.hoya.yarn.client.HoyaYarnClientImpl.java
License:Apache License
/** * Helper method to determine if a cluster application is running -or * is earlier in the lifecycle/*ww w. ja v a 2s .c o m*/ * @param app application report * @return true if the application is considered live */ public boolean isApplicationLive(ApplicationReport app) { return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal(); }
From source file:org.apache.hoya.yarn.client.HoyaYarnClientImpl.java
License:Apache License
/** * Monitor the submitted application for reaching the requested state. * Will also report if the app reaches a later state (failed, killed, etc) * Kill application if duration!= null & time expires. * @param appId Application Id of application to be monitored * @param duration how long to wait -must be more than 0 * @param desiredState desired state.//from w w w. j a v a 2s . c o m * @return the application report -null on a timeout * @throws YarnException * @throws IOException */ public ApplicationReport monitorAppToState(ApplicationId appId, YarnApplicationState desiredState, Duration duration) throws YarnException, IOException { if (appId == null) { throw new BadCommandArgumentsException("null application ID"); } if (duration.limit <= 0) { throw new BadCommandArgumentsException("Invalid monitoring duration"); } log.debug("Waiting {} millis for app to reach state {} ", duration.limit, desiredState); duration.start(); while (true) { // Get application report for the appId we are interested in ApplicationReport r = getApplicationReport(appId); log.debug("queried status is\n{}", new HoyaUtils.OnDemandReportStringifier(r)); YarnApplicationState state = r.getYarnApplicationState(); if (state.ordinal() >= desiredState.ordinal()) { log.debug("App in desired state (or higher) :{}", state); return r; } if (duration.getLimitExceeded()) { log.debug("Wait limit of {} millis to get to state {}, exceeded; app status\n {}", duration.limit, desiredState, new HoyaUtils.OnDemandReportStringifier(r)); return null; } // sleep 1s. try { Thread.sleep(1000); } catch (InterruptedException ignored) { log.debug("Thread sleep in monitoring loop interrupted"); } } }
From source file:org.apache.ignite.yarn.IgniteYarnClient.java
License:Apache License
/** * Main methods has one mandatory parameter and one optional parameter. * * @param args Path to jar mandatory parameter and property file is optional. *//*from w ww .j a v a 2s . co m*/ public static void main(String[] args) throws Exception { checkArguments(args); // Set path to app master jar. String pathAppMasterJar = args[0]; ClusterProperties props = ClusterProperties.from(args.length == 2 ? args[1] : null); YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); FileSystem fs = FileSystem.get(conf); Path ignite; // Load ignite and jar if (props.ignitePath() == null) ignite = getIgnite(props, fs); else ignite = new Path(props.ignitePath()); // Upload the jar file to HDFS. Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar, props.igniteWorkDir() + File.separator + IgniteYarnUtils.JAR_NAME); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands(Collections .singletonList(Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName() + IgniteYarnUtils.SPACE + ignite.toUri() + IgniteYarnUtils.YARN_LOG_OUT)); // Setup jar for ApplicationMaster LocalResource appMasterJar = IgniteYarnUtils.setupFile(appJar, fs, LocalResourceType.FILE); amContainer.setLocalResources(Collections.singletonMap(IgniteYarnUtils.JAR_NAME, appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = props.toEnvs(); setupAppMasterEnv(appMasterEnv, conf); amContainer.setEnvironment(appMasterEnv); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials creds = new Credentials(); String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokRenewer == null || tokRenewer.length() == 0) throw new IOException("Master Kerberos principal for the RM is not set."); log.info("Found RM principal: " + tokRenewer); final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds); if (tokens != null) log.info("File system delegation tokens: " + Arrays.toString(tokens)); amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds)); } // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(512); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("ignition"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); yarnClient.submitApplication(appContext); log.log(Level.INFO, "Submitted application. Application id: {0}", appId); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED || appState == YarnApplicationState.ACCEPTED) { TimeUnit.SECONDS.sleep(1L); appReport = yarnClient.getApplicationReport(appId); if (appState != YarnApplicationState.ACCEPTED && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) log.log(Level.INFO, "Application {0} is ACCEPTED.", appId); appState = appReport.getYarnApplicationState(); } log.log(Level.INFO, "Application {0} is {1}.", new Object[] { appId, appState }); }
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
/** * Monitor the submitted application for completion. * Kill application if time expires./*from w w w . j a va2 s.co m*/ * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.RUNNING == state) { LOG.info("Application is running..."); return true; } if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
From source file:org.apache.metron.maas.service.MaasIntegrationTest.java
License:Apache License
public void testDSShell(boolean haveDomain) throws Exception { MaaSConfig config = new MaaSConfig() { {/*from w w w.ja va 2 s . c o m*/ setServiceRoot("/maas/service"); setQueueConfig(new HashMap<String, Object>() { { put(ZKQueue.ZK_PATH, "/maas/queue"); } }); } }; String configRoot = "/maas/config"; byte[] configData = ConfigUtil.INSTANCE.toBytes(config); try { client.setData().forPath(configRoot, configData); } catch (KeeperException.NoNodeException e) { client.create().creatingParentsIfNeeded().forPath(configRoot, configData); } String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2", }; if (haveDomain) { String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" }; List<String> argsList = new ArrayList<String>(Arrays.asList(args)); argsList.addAll(Arrays.asList(domainArgs)); args = argsList.toArray(new String[argsList.size()]); } YarnConfiguration conf = yarnComponent.getConfig(); LOG.info("Initializing DS Client"); final Client client = new Client(new Configuration(conf)); boolean initSuccess = client.init(args); Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread() { @Override public void run() { try { result.set(client.run()); } catch (Exception e) { throw new RuntimeException(e); } } }; t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); boolean verified = false; String errorMessage = ""; while (!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0) { Thread.sleep(10); continue; } ApplicationReport appReport = apps.get(0); if (appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; } errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break; } } Assert.assertTrue(errorMessage, verified); FileSystem fs = FileSystem.get(conf); try { new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" }); ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot()); discoverer.start(); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); if (endpoints != null && endpoints.size() == 1) { LOG.trace("Found endpoints: " + endpoints.get(0)); String output = makeRESTcall( new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey")); if (output.contains("casey")) { passed = true; break; } } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); Assert.assertNotNull(endpoints); Assert.assertEquals(1, endpoints.size()); } new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE", }); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); //ensure that the endpoint is dead. if (endpoints == null || endpoints.size() == 0) { passed = true; break; } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } } finally { cleanup(); } }
From source file:org.apache.reef.runtime.yarn.client.unmanaged.UnmanagedAmYarnSubmissionHelper.java
License:Apache License
@Override public void close() { if (LOG.isLoggable(Level.FINER)) { try {/*from ww w . j av a2 s .c o m*/ final ApplicationReport appReport = this.yarnClient.getApplicationReport(this.applicationId); LOG.log(Level.FINER, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { this.applicationId, appReport.getCurrentApplicationAttemptId(), appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus() }); } catch (final IOException | YarnException ex) { LOG.log(Level.WARNING, "Cannot get final status of Unmanaged AM app: " + this.applicationId, ex); } } LOG.log(Level.FINE, "Closing Unmanaged AM YARN application: {0}", this.applicationId); this.yarnClient.stop(); }
From source file:org.apache.reef.runtime.yarn.driver.unmanaged.UnmanagedAmTest.java
License:Apache License
@Test public void testAmShutdown() throws IOException, YarnException { Assume.assumeTrue("This test requires a YARN Resource Manager to connect to", Boolean.parseBoolean(System.getenv("REEF_TEST_YARN"))); final YarnConfiguration yarnConfig = new YarnConfiguration(); // Start YARN client and register the application final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfig);// w ww. j av a2s .c o m yarnClient.start(); final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class); containerContext.setCommands(Collections.<String>emptyList()); containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap()); containerContext.setEnvironment(Collections.<String, String>emptyMap()); containerContext.setTokens(getTokens()); final ApplicationSubmissionContext appContext = yarnClient.createApplication() .getApplicationSubmissionContext(); appContext.setApplicationName("REEF_Unmanaged_AM_Test"); appContext.setAMContainerSpec(containerContext); appContext.setUnmanagedAM(true); appContext.setQueue("default"); final ApplicationId applicationId = appContext.getApplicationId(); LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId); yarnClient.submitApplication(appContext); LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId); addToken(yarnClient.getAMRMToken(applicationId)); // Start the AM final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this); rmClient.init(yarnConfig); rmClient.start(); final NMClientAsync nmClient = new NMClientAsyncImpl(this); nmClient.init(yarnConfig); nmClient.start(); final RegisterApplicationMasterResponse registration = rmClient .registerApplicationMaster(NetUtils.getHostname(), -1, null); LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null); LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState()); // Shutdown the AM rmClient.stop(); nmClient.stop(); // Get the final application report final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); final YarnApplicationState appState = appReport.getYarnApplicationState(); final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus(); LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus }); Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState); Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus); // Shutdown YARN client yarnClient.stop(); }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * Wait for the launched app to be accepted in the time * and, optionally running.//from w w w. j av a 2 s . c o m * <p> * If the application * * @param launchedApplication application * @param acceptWaitMillis time in millis to wait for accept * @param runWaitMillis time in millis to wait for the app to be running. * May be null, in which case no wait takes place * @return exit code: success * @throws YarnException * @throws IOException */ public int waitForAppRunning(LaunchedApplication launchedApplication, int acceptWaitMillis, int runWaitMillis) throws YarnException, IOException { assert launchedApplication != null; int exitCode; // wait for the submit state to be reached ApplicationReport report = launchedApplication.monitorAppToState(YarnApplicationState.ACCEPTED, new Duration(acceptWaitMillis)); // may have failed, so check that if (SliderUtils.hasAppFinished(report)) { exitCode = buildExitCode(report); } else { // exit unless there is a wait if (runWaitMillis != 0) { // waiting for state to change Duration duration = new Duration(runWaitMillis * 1000); duration.start(); report = launchedApplication.monitorAppToState(YarnApplicationState.RUNNING, duration); if (report != null && report.getYarnApplicationState() == YarnApplicationState.RUNNING) { exitCode = EXIT_SUCCESS; } else { exitCode = buildExitCode(report); } } else { exitCode = EXIT_SUCCESS; } } return exitCode; }