List of usage examples for org.apache.hadoop.yarn.client.api YarnClient getApplications
public abstract List<ApplicationReport> getApplications() throws YarnException, IOException;
Get a report (ApplicationReport) of all Applications in the cluster.
From source file:co.cask.cdap.operations.yarn.YarnApps.java
License:Apache License
@Override public synchronized void collect() throws Exception { reset();/*from w w w . j a v a 2s . c o m*/ YarnClient yarnClient = createYARNClient(); List<ApplicationReport> applications; try { applications = yarnClient.getApplications(); } finally { yarnClient.stop(); } for (ApplicationReport application : applications) { switch (application.getYarnApplicationState()) { case NEW: case NEW_SAVING: newApps++; break; case ACCEPTED: accepted++; break; case SUBMITTED: submitted++; break; case RUNNING: running++; break; case FINISHED: finished++; break; case FAILED: failed++; break; case KILLED: killed++; break; } } total = applications.size(); }
From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java
License:Apache License
/** * Test to verify Llama deletes old reservations on startup. *///from w w w.j av a 2 s .co m @Test(timeout = 60000) public void testLlamaDeletesOldReservationsOnStartup() throws Exception { YarnClient client = null; LlamaAM llamaAM1 = null, llamaAM2 = null, llamaAM3 = null; EnumSet<YarnApplicationState> running = EnumSet.of(YarnApplicationState.RUNNING); try { startYarn(createMiniYarnConfig(false)); client = YarnClient.createYarnClient(); client.init(miniYarn.getConfig()); client.start(); Assert.assertEquals("Non-zero YARN apps even before any reservations", 0, client.getApplications().size()); llamaAM1 = LlamaAM.create(getLlamaConfiguration()); llamaAM1.start(); Assert.assertEquals("Mismatch between #YARN apps and #Queues", 2, client.getApplications(running).size()); // Start another Llama of the same cluster-id to see if old YARN apps // are deleted. llamaAM2 = LlamaAM.create(getLlamaConfiguration()); llamaAM2.start(); Assert.assertEquals( "Mismatch between #YARN apps and #Queues. Only apps" + " from the latest started Llama should be running.", 2, client.getApplications(running).size()); // Start Llama of different cluster-id to see old YARN apps are not // deleted. Configuration confWithDifferentCluserId = getLlamaConfiguration(); confWithDifferentCluserId.set(LlamaAM.CLUSTER_ID, "new-cluster"); llamaAM3 = LlamaAM.create(confWithDifferentCluserId); llamaAM3.start(); Assert.assertEquals("Mismatch between #YARN apps and #Queues for " + "multiple clusters", 4, client.getApplications(running).size()); } finally { client.stop(); llamaAM1.stop(); llamaAM2.stop(); llamaAM3.stop(); stopYarn(); } }
From source file:org.apache.metron.maas.service.MaasIntegrationTest.java
License:Apache License
public void testDSShell(boolean haveDomain) throws Exception { MaaSConfig config = new MaaSConfig() { {/*from w w w . j av a 2s . c om*/ setServiceRoot("/maas/service"); setQueueConfig(new HashMap<String, Object>() { { put(ZKQueue.ZK_PATH, "/maas/queue"); } }); } }; String configRoot = "/maas/config"; byte[] configData = ConfigUtil.INSTANCE.toBytes(config); try { client.setData().forPath(configRoot, configData); } catch (KeeperException.NoNodeException e) { client.create().creatingParentsIfNeeded().forPath(configRoot, configData); } String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2", }; if (haveDomain) { String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" }; List<String> argsList = new ArrayList<String>(Arrays.asList(args)); argsList.addAll(Arrays.asList(domainArgs)); args = argsList.toArray(new String[argsList.size()]); } YarnConfiguration conf = yarnComponent.getConfig(); LOG.info("Initializing DS Client"); final Client client = new Client(new Configuration(conf)); boolean initSuccess = client.init(args); Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread() { @Override public void run() { try { result.set(client.run()); } catch (Exception e) { throw new RuntimeException(e); } } }; t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); boolean verified = false; String errorMessage = ""; while (!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0) { Thread.sleep(10); continue; } ApplicationReport appReport = apps.get(0); if (appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; } errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break; } } Assert.assertTrue(errorMessage, verified); FileSystem fs = FileSystem.get(conf); try { new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" }); ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot()); discoverer.start(); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); if (endpoints != null && endpoints.size() == 1) { LOG.trace("Found endpoints: " + endpoints.get(0)); String output = makeRESTcall( new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey")); if (output.contains("casey")) { passed = true; break; } } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); Assert.assertNotNull(endpoints); Assert.assertEquals(1, endpoints.size()); } new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE", }); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); //ensure that the endpoint is dead. if (endpoints == null || endpoints.size() == 0) { passed = true; break; } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } } finally { cleanup(); } }
From source file:org.apache.tez.test.MiniTezCluster.java
License:Apache License
private void waitForAppsToFinish() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(getConfig());// w w w .ja va 2 s . com yarnClient.start(); try { while (true) { List<ApplicationReport> appReports = yarnClient.getApplications(); Collection<ApplicationReport> unCompletedApps = Collections2.filter(appReports, new Predicate<ApplicationReport>() { @Override public boolean apply(ApplicationReport appReport) { return EnumSet .of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) .contains(appReport.getYarnApplicationState()); } }); if (unCompletedApps.size() == 0) { break; } LOG.info("wait for applications to finish in MiniTezCluster"); Thread.sleep(1000); } } catch (Exception e) { e.printStackTrace(); } finally { yarnClient.stop(); } }
From source file:org.apache.tez.test.TestTezJobs.java
License:Apache License
@Test(timeout = 60000) public void testSimpleSessionExample() throws Exception { Path stagingDirPath = new Path("/tmp/owc-staging-dir"); remoteFs.mkdirs(stagingDirPath);// w w w . j a v a 2 s . c o m int numIterations = 2; String[] inputPaths = new String[numIterations]; String[] outputPaths = new String[numIterations]; Path[] outputDirs = new Path[numIterations]; for (int i = 0; i < numIterations; ++i) { String inputDirStr = "/tmp/owc-input-" + i + "/"; inputPaths[i] = inputDirStr; Path inputDir = new Path(inputDirStr); remoteFs.mkdirs(inputDir); generateOrderedWordCountInput(inputDir, remoteFs); String outputDirStr = "/tmp/owc-output-" + i + "/"; outputPaths[i] = outputDirStr; Path outputDir = new Path(outputDirStr); outputDirs[i] = outputDir; } TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString()); YarnClient yarnClient = YarnClient.createYarnClient(); try { yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); List<ApplicationReport> apps = yarnClient.getApplications(); int appsBeforeCount = apps != null ? apps.size() : 0; SimpleSessionExample job = new SimpleSessionExample(); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); Assert.assertTrue("SimpleSessionExample failed", job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null) == 0); for (int i = 0; i < numIterations; ++i) { verifyOutput(outputDirs[i], remoteFs); } apps = yarnClient.getApplications(); int appsAfterCount = apps != null ? apps.size() : 0; // Running in session mode. So should only create 1 more app. Assert.assertEquals(appsBeforeCount + 1, appsAfterCount); } finally { remoteFs.delete(stagingDirPath, true); if (yarnClient != null) { yarnClient.stop(); } } }
From source file:org.apache.tez.tests.MiniTezClusterWithTimeline.java
License:Apache License
private void waitForAppsToFinish() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(getConfig());/*from w w w .j a v a2 s . c o m*/ yarnClient.start(); try { while (true) { List<ApplicationReport> appReports = yarnClient.getApplications(); Collection<ApplicationReport> unCompletedApps = Collections2.filter(appReports, new Predicate<ApplicationReport>() { @Override public boolean apply(ApplicationReport appReport) { return EnumSet .of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) .contains(appReport.getYarnApplicationState()); } }); if (unCompletedApps.size() == 0) { break; } LOG.info("wait for applications to finish in MiniTezClusterWithTimeline"); Thread.sleep(1000); } } catch (Exception e) { e.printStackTrace(); } finally { yarnClient.stop(); } }
From source file:yrun.commands.Yps.java
License:Apache License
public static void main(String[] args) throws YarnException, IOException { YarnClient yarnClient = YarnClient.createYarnClient(); YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnClient.init(yarnConfiguration);//from ww w. j a v a2s.c o m yarnClient.start(); try { List<ApplicationReport> applications = yarnClient.getApplications(); for (ApplicationReport applicationReport : applications) { ApplicationId applicationId = applicationReport.getApplicationId(); String user = applicationReport.getUser(); String queue = applicationReport.getQueue(); String name = applicationReport.getName(); YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); float progress = applicationReport.getProgress(); System.out.printf("%s\t%s\t%s\t%s\t%s\t%f%n", toString(applicationId), user, queue, name, yarnApplicationState.name(), progress); } } finally { yarnClient.stop(); yarnClient.close(); } }