List of usage examples for org.apache.hadoop.yarn.api.records YarnApplicationState FINISHED
YarnApplicationState FINISHED
To view the source code for org.apache.hadoop.yarn.api.records YarnApplicationState FINISHED.
Click Source Link
From source file:org.apache.metron.maas.service.MaasIntegrationTest.java
License:Apache License
public void testDSShell(boolean haveDomain) throws Exception { MaaSConfig config = new MaaSConfig() { {/*w w w . j a v a2 s .c om*/ setServiceRoot("/maas/service"); setQueueConfig(new HashMap<String, Object>() { { put(ZKQueue.ZK_PATH, "/maas/queue"); } }); } }; String configRoot = "/maas/config"; byte[] configData = ConfigUtil.INSTANCE.toBytes(config); try { client.setData().forPath(configRoot, configData); } catch (KeeperException.NoNodeException e) { client.create().creatingParentsIfNeeded().forPath(configRoot, configData); } String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2", }; if (haveDomain) { String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" }; List<String> argsList = new ArrayList<String>(Arrays.asList(args)); argsList.addAll(Arrays.asList(domainArgs)); args = argsList.toArray(new String[argsList.size()]); } YarnConfiguration conf = yarnComponent.getConfig(); LOG.info("Initializing DS Client"); final Client client = new Client(new Configuration(conf)); boolean initSuccess = client.init(args); Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread() { @Override public void run() { try { result.set(client.run()); } catch (Exception e) { throw new RuntimeException(e); } } }; t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); boolean verified = false; String errorMessage = ""; while (!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0) { Thread.sleep(10); continue; } ApplicationReport appReport = apps.get(0); if (appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; } errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break; } } Assert.assertTrue(errorMessage, verified); FileSystem fs = FileSystem.get(conf); try { new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" }); ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot()); discoverer.start(); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); if (endpoints != null && endpoints.size() == 1) { LOG.trace("Found endpoints: " + endpoints.get(0)); String output = makeRESTcall( new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey")); if (output.contains("casey")) { passed = true; break; } } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); Assert.assertNotNull(endpoints); Assert.assertEquals(1, endpoints.size()); } new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE", }); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); //ensure that the endpoint is dead. if (endpoints == null || endpoints.size() == 0) { passed = true; break; } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } } finally { cleanup(); } }
From source file:org.apache.reef.runtime.yarn.driver.unmanaged.UnmanagedAmTest.java
License:Apache License
@Test public void testAmShutdown() throws IOException, YarnException { Assume.assumeTrue("This test requires a YARN Resource Manager to connect to", Boolean.parseBoolean(System.getenv("REEF_TEST_YARN"))); final YarnConfiguration yarnConfig = new YarnConfiguration(); // Start YARN client and register the application final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfig);/*from ww w . j a v a2s.co m*/ yarnClient.start(); final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class); containerContext.setCommands(Collections.<String>emptyList()); containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap()); containerContext.setEnvironment(Collections.<String, String>emptyMap()); containerContext.setTokens(getTokens()); final ApplicationSubmissionContext appContext = yarnClient.createApplication() .getApplicationSubmissionContext(); appContext.setApplicationName("REEF_Unmanaged_AM_Test"); appContext.setAMContainerSpec(containerContext); appContext.setUnmanagedAM(true); appContext.setQueue("default"); final ApplicationId applicationId = appContext.getApplicationId(); LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId); yarnClient.submitApplication(appContext); LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId); addToken(yarnClient.getAMRMToken(applicationId)); // Start the AM final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this); rmClient.init(yarnConfig); rmClient.start(); final NMClientAsync nmClient = new NMClientAsyncImpl(this); nmClient.init(yarnConfig); nmClient.start(); final RegisterApplicationMasterResponse registration = rmClient .registerApplicationMaster(NetUtils.getHostname(), -1, null); LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null); LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState()); // Shutdown the AM rmClient.stop(); nmClient.stop(); // Get the final application report final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); final YarnApplicationState appState = appReport.getYarnApplicationState(); final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus(); LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus }); Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState); Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus); // Shutdown YARN client yarnClient.stop(); }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException { verifyBindingsDefined();/* w w w . j a va2 s .c o m*/ SliderUtils.validateClusterName(name); boolean checkLive = args.live; log.debug("actionExists({}, {}, {})", name, checkLive, args.state); //initial probe for a cluster in the filesystem Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name); if (!sliderFileSystem.getFileSystem().exists(clusterDirectory)) { throw unknownClusterException(name); } String state = args.state; if (!checkLive && SliderUtils.isUnset(state)) { log.info("Application {} exists", name); return EXIT_SUCCESS; } //test for liveness/state boolean inDesiredState = false; ApplicationReport instance; instance = findInstance(name); if (instance == null) { log.info("Application {} not running", name); return EXIT_FALSE; } if (checkLive) { // the app exists, check that it is not in any terminated state YarnApplicationState appstate = instance.getYarnApplicationState(); log.debug(" current app state = {}", appstate); inDesiredState = appstate.ordinal() < YarnApplicationState.FINISHED.ordinal(); } else { // scan for instance in single --state state List<ApplicationReport> userInstances = yarnClient.listInstances(""); state = state.toUpperCase(Locale.ENGLISH); YarnApplicationState desiredState = extractYarnApplicationState(state); ApplicationReport foundInstance = yarnClient.findAppInInstanceList(userInstances, name, desiredState); if (foundInstance != null) { // found in selected state: success inDesiredState = true; // mark this as the instance to report instance = foundInstance; } } SliderUtils.OnDemandReportStringifier report = new SliderUtils.OnDemandReportStringifier(instance); if (!inDesiredState) { //cluster in the list of apps but not running log.info("Application {} found but is in wrong state {}", name, instance.getYarnApplicationState()); log.debug("State {}", report); return EXIT_FALSE; } else { log.debug("Application instance is in desired state"); log.info("Application {} is {}\n{}", name, instance.getYarnApplicationState(), report); return EXIT_SUCCESS; } }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
@Override public int actionFreeze(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException { verifyBindingsDefined();//from w ww. ja v a2s . co m SliderUtils.validateClusterName(clustername); int waittime = freezeArgs.getWaittime(); String text = freezeArgs.message; boolean forcekill = freezeArgs.force; log.debug("actionFreeze({}, reason={}, wait={}, force={})", clustername, text, waittime, forcekill); //is this actually a known cluster? sliderFileSystem.locateInstanceDefinition(clustername); ApplicationReport app = findInstance(clustername); if (app == null) { // exit early log.info("Cluster {} not running", clustername); // not an error to stop a stopped cluster return EXIT_SUCCESS; } log.debug("App to stop was found: {}:\n{}", clustername, new SliderUtils.OnDemandReportStringifier(app)); if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) { log.info("Cluster {} is a terminated state {}", clustername, app.getYarnApplicationState()); return EXIT_SUCCESS; } // IPC request for a managed shutdown is only possible if the app is running. // so we need to force kill if the app is accepted or submitted if (!forcekill && app.getYarnApplicationState().ordinal() < YarnApplicationState.RUNNING.ordinal()) { log.info("Cluster {} is in a pre-running state {}. Force killing it", clustername, app.getYarnApplicationState()); forcekill = true; } LaunchedApplication application = new LaunchedApplication(yarnClient, app); applicationId = application.getApplicationId(); if (forcekill) { // escalating to forced kill application.kill("Forced stop of " + clustername + ": " + text); } else { try { SliderClusterProtocol appMaster = connect(app); Messages.StopClusterRequestProto r = Messages.StopClusterRequestProto.newBuilder().setMessage(text) .build(); appMaster.stopCluster(r); log.debug("Cluster stop command issued"); } catch (YarnException e) { log.warn("Exception while trying to terminate {}: {}", clustername, e); return EXIT_FALSE; } catch (IOException e) { log.warn("Exception while trying to terminate {}: {}", clustername, e); return EXIT_FALSE; } } //wait for completion. We don't currently return an exception during this process //as the stop operation has been issued, this is just YARN. try { if (waittime > 0) { ApplicationReport applicationReport = application.monitorAppToState(YarnApplicationState.FINISHED, new Duration(waittime * 1000)); if (applicationReport == null) { log.info("application did not shut down in time"); return EXIT_FALSE; } } // JDK7 } catch (YarnException | IOException e) { } catch (YarnException e) { log.warn("Exception while waiting for the application {} to shut down: {}", clustername, e); } catch (IOException e) { log.warn("Exception while waiting for the application {} to shut down: {}", clustername, e); } return EXIT_SUCCESS; }
From source file:org.apache.slider.common.tools.TestSliderUtils.java
License:Apache License
private List<ApplicationReport> getApplicationReports() { List<ApplicationReport> instances = new ArrayList<ApplicationReport>(); instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED)); instances.add(getApplicationReport(900, 998, "app1", YarnApplicationState.KILLED)); instances.add(getApplicationReport(900, 998, "app2", YarnApplicationState.FAILED)); instances.add(getApplicationReport(1000, 0, "app2", YarnApplicationState.RUNNING)); instances.add(getApplicationReport(800, 837, "app3", YarnApplicationState.FINISHED)); instances.add(getApplicationReport(1000, 0, "app3", YarnApplicationState.RUNNING)); instances.add(getApplicationReport(900, 998, "app3", YarnApplicationState.KILLED)); instances.add(getApplicationReport(800, 837, "app4", YarnApplicationState.FINISHED)); instances.add(getApplicationReport(1000, 1050, "app4", YarnApplicationState.KILLED)); instances.add(getApplicationReport(900, 998, "app4", YarnApplicationState.FINISHED)); Assert.assertEquals("app1", instances.get(0).getApplicationType()); Assert.assertEquals("app1", instances.get(1).getApplicationType()); Assert.assertEquals("app2", instances.get(2).getApplicationType()); Assert.assertEquals("app2", instances.get(3).getApplicationType()); return instances; }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
public boolean isQueryMasterStopped(QueryId queryId) { ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId); try {/*from www.j av a 2 s . c o m*/ ApplicationReport report = yarnClient.getApplicationReport(appId); YarnApplicationState state = report.getYarnApplicationState(); return EnumSet .of(YarnApplicationState.FINISHED, YarnApplicationState.KILLED, YarnApplicationState.FAILED) .contains(state); } catch (YarnException e) { LOG.error(e.getMessage(), e); return false; } catch (IOException e) { LOG.error(e.getMessage(), e); return false; } }
From source file:org.apache.tez.auxservices.TestShuffleHandlerJobs.java
License:Apache License
@Test(timeout = 300000) public void testOrderedWordCount() throws Exception { String inputDirStr = "/tmp/owc-input/"; Path inputDir = new Path(inputDirStr); Path stagingDirPath = new Path("/tmp/owc-staging-dir"); remoteFs.mkdirs(inputDir);/*from w w w. ja v a 2 s .co m*/ remoteFs.mkdirs(stagingDirPath); generateOrderedWordCountInput(inputDir, remoteFs); String outputDirStr = "/tmp/owc-output/"; Path outputDir = new Path(outputDirStr); TezConfiguration tezConf = new TezConfiguration(tezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString()); tezConf.set(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, ShuffleHandler.TEZ_SHUFFLE_SERVICEID); tezConf.setBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION, true); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); tezConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, false); tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); TezClient tezSession = TezClient.create("WordCountTest", tezConf); tezSession.start(); try { final OrderedWordCount job = new OrderedWordCount(); Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[] { "-counter", inputDirStr, outputDirStr, "10" }, tezSession) == 0); verifyOutput(outputDir, remoteFs); tezSession.stop(); ClientRMService rmService = tezCluster.getResourceManager().getClientRMService(); boolean isAppComplete = false; while (!isAppComplete) { GetApplicationReportResponse resp = rmService .getApplicationReport(new GetApplicationReportRequest() { @Override public ApplicationId getApplicationId() { return job.getAppId(); } @Override public void setApplicationId(ApplicationId applicationId) { } }); if (resp.getApplicationReport().getYarnApplicationState() == YarnApplicationState.FINISHED) { isAppComplete = true; } Thread.sleep(100); } for (int i = 0; i < NUM_NMS; i++) { String appPath = tezCluster.getTestWorkDir() + "/" + this.getClass().getName() + "-localDir-nm-" + i + "_0/usercache/" + UserGroupInformation.getCurrentUser().getUserName() + "/appcache/" + job.getAppId(); String dagPathStr = appPath + "/dag_1"; File fs = new File(dagPathStr); Assert.assertFalse(fs.exists()); fs = new File(appPath); Assert.assertTrue(fs.exists()); } } finally { remoteFs.delete(stagingDirPath, true); } }
From source file:org.apache.tez.client.LocalClient.java
License:Apache License
protected YarnApplicationState convertDAGAppMasterState(DAGAppMasterState dagAppMasterState) { switch (dagAppMasterState) { case NEW:// w ww .ja va2 s.c o m return YarnApplicationState.NEW; case INITED: case RECOVERING: case IDLE: case RUNNING: return YarnApplicationState.RUNNING; case SUCCEEDED: return YarnApplicationState.FINISHED; case FAILED: return YarnApplicationState.FAILED; case KILLED: return YarnApplicationState.KILLED; case ERROR: return YarnApplicationState.FAILED; default: return YarnApplicationState.SUBMITTED; } }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
static DAGClientAMProtocolBlockingPB getSessionAMProxy(FrameworkClient yarnClient, Configuration conf, ApplicationId applicationId) throws TezException, IOException { ApplicationReport appReport;/*w ww .j a v a 2 s. co m*/ try { appReport = yarnClient.getApplicationReport(applicationId); if (appReport == null) { throw new TezUncheckedException( "Could not retrieve application report" + " from YARN, applicationId=" + applicationId); } YarnApplicationState appState = appReport.getYarnApplicationState(); if (appState != YarnApplicationState.RUNNING) { if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) { String msg = "Application not running" + ", applicationId=" + applicationId + ", yarnApplicationState=" + appReport.getYarnApplicationState() + ", finalApplicationStatus=" + appReport.getFinalApplicationStatus() + ", trackingUrl=" + appReport.getTrackingUrl() + ", diagnostics=" + (appReport.getDiagnostics() != null ? appReport.getDiagnostics() : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG); LOG.info(msg); throw new SessionNotRunning(msg); } return null; } } catch (YarnException e) { throw new TezException(e); } return getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(), appReport.getClientToAMToken()); }
From source file:org.apache.tez.dag.api.client.DAGClientImpl.java
License:Apache License
private void checkAndSetDagCompletionStatus() { ApplicationReport appReport = realClient.getApplicationReportInternal(); if (appReport != null) { final YarnApplicationState appState = appReport.getYarnApplicationState(); if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED) { dagCompleted = true;/*from w w w . ja v a 2 s . c om*/ } } }