List of usage examples for org.apache.hadoop.yarn.client.api YarnClient start
@Override public void start()
From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java
License:Apache License
private static YarnClient createYarnClient() { YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);//from w ww . java2s. com yarnClient.start(); return yarnClient; }
From source file:org.apache.tez.mapreduce.examples.BroadcastAndOneToOneExample.java
License:Apache License
private DAG createDAG(FileSystem fs, TezConfiguration tezConf, Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException { int numBroadcastTasks = 2; int numOneToOneTasks = 3; if (doLocalityCheck) { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(tezConf);/*from w w w . j ava2 s.com*/ yarnClient.start(); int numNMs = yarnClient.getNodeReports(NodeState.RUNNING).size(); yarnClient.stop(); // create enough 1-1 tasks to run in parallel numOneToOneTasks = numNMs - numBroadcastTasks - 1;// 1 AM if (numOneToOneTasks < 1) { numOneToOneTasks = 1; } } byte[] procByte = { (byte) (doLocalityCheck ? 1 : 0), 1 }; UserPayload procPayload = UserPayload.create(ByteBuffer.wrap(procByte)); System.out.println("Using " + numOneToOneTasks + " 1-1 tasks"); Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create(InputProcessor.class.getName()), numBroadcastTasks); Vertex inputVertex = Vertex.create("Input", ProcessorDescriptor.create(InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks); Vertex oneToOneVertex = Vertex.create("OneToOne", ProcessorDescriptor.create(OneToOneProcessor.class.getName()).setUserPayload(procPayload)); oneToOneVertex.setVertexManagerPlugin( VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName())); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig .newBuilder(Text.class.getName(), IntWritable.class.getName()).setFromConfiguration(tezConf) .build(); DAG dag = DAG.create("BroadcastAndOneToOneExample"); dag.addVertex(inputVertex).addVertex(broadcastVertex).addVertex(oneToOneVertex) .addEdge(Edge.create(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty())) .addEdge(Edge.create(broadcastVertex, oneToOneVertex, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; }
From source file:org.apache.tez.mapreduce.examples.OrderedWordCount.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: wordcount <in> <out>"); System.exit(2);// www . java 2 s . c om } // Configure intermediate reduces conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1); // Set reducer class for intermediate reduce conf.setClass(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduce.class"), IntSumReducer.class, Reducer.class); // Set reducer output key class conf.setClass( MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.map.output.key.class"), IntWritable.class, Object.class); // Set reducer output value class conf.setClass( MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.map.output.value.class"), Text.class, Object.class); conf.setInt(MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduces"), 2); @SuppressWarnings("deprecation") Job job = new Job(conf, "orderedwordcount"); job.setJarByClass(OrderedWordCount.class); // Configure map job.setMapperClass(TokenizerMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); // Configure reduce job.setReducerClass(MyOrderByNoOpReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); YarnClient yarnClient = new YarnClientImpl(); yarnClient.init(conf); yarnClient.start(); TezClient tezClient = new TezClient(new TezConfiguration(conf)); job.submit(); JobID jobId = job.getJobID(); ApplicationId appId = TypeConverter.toYarn(jobId).getAppId(); DAGClient dagClient = tezClient.getDAGClient(appId); DAGStatus dagStatus = null; while (true) { dagStatus = dagClient.getDAGStatus(); if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { break; } try { Thread.sleep(500); } catch (InterruptedException e) { // continue; } } while (dagStatus.getState() == DAGStatus.State.RUNNING) { try { ExampleDriver.printMRRDAGStatus(dagStatus); try { Thread.sleep(1000); } catch (InterruptedException e) { // continue; } dagStatus = dagClient.getDAGStatus(); } catch (TezException e) { LOG.fatal("Failed to get application progress. Exiting"); System.exit(-1); } } ExampleDriver.printMRRDAGStatus(dagStatus); LOG.info("Application completed. " + "FinalState=" + dagStatus.getState()); System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1); }
From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java
License:Apache License
private void stopAndVerifyYarnApp(TezClient tezSession) throws TezException, IOException, YarnException { ApplicationId appId = tezSession.getAppMasterApplicationId(); tezSession.stop();/*from w w w . ja v a 2s .com*/ Assert.assertEquals(TezAppMasterStatus.SHUTDOWN, tezSession.getAppMasterStatus()); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState().equals(YarnApplicationState.FINISHED) || appReport.getYarnApplicationState().equals(YarnApplicationState.FAILED) || appReport.getYarnApplicationState().equals(YarnApplicationState.KILLED)) { break; } } ApplicationReport appReport = yarnClient.getApplicationReport(appId); Assert.assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState()); Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus()); }
From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java
License:Apache License
public State testMRRSleepJobDagSubmitCore(boolean dagViaRPC, boolean killDagWhileRunning, boolean closeSessionBeforeSubmit, TezClient reUseTezSession, boolean genSplitsInAM, Class<? extends InputInitializer> initializerClass, Map<String, LocalResource> additionalLocalResources) throws IOException, InterruptedException, TezException, ClassNotFoundException, YarnException { LOG.info("\n\n\nStarting testMRRSleepJobDagSubmit()."); JobConf stage1Conf = new JobConf(mrrTezCluster.getConfig()); JobConf stage2Conf = new JobConf(mrrTezCluster.getConfig()); JobConf stage3Conf = new JobConf(mrrTezCluster.getConfig()); stage1Conf.setLong(MRRSleepJob.MAP_SLEEP_TIME, 1); stage1Conf.setInt(MRRSleepJob.MAP_SLEEP_COUNT, 1); stage1Conf.setInt(MRJobConfig.NUM_MAPS, 1); stage1Conf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName()); stage1Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); stage1Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName()); stage1Conf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, SleepInputFormat.class.getName()); stage1Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName()); stage2Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1); stage2Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1); stage2Conf.setInt(MRJobConfig.NUM_REDUCES, 1); stage2Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName()); stage2Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); stage2Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName()); stage2Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName()); stage3Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1); stage3Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1); stage3Conf.setInt(MRJobConfig.NUM_REDUCES, 1); stage3Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName()); stage3Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); stage3Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName()); MRHelpers.translateMRConfToTez(stage1Conf); MRHelpers.translateMRConfToTez(stage2Conf); MRHelpers.translateMRConfToTez(stage3Conf); MRHelpers.configureMRApiUsage(stage1Conf); MRHelpers.configureMRApiUsage(stage2Conf); MRHelpers.configureMRApiUsage(stage3Conf); Path remoteStagingDir = remoteFs .makeQualified(new Path("/tmp", String.valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf); UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf); DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000)); Class<? extends InputInitializer> inputInitializerClazz = genSplitsInAM ? (initializerClass == null ? MRInputAMSplitGenerator.class : initializerClass) : null;/*from ww w.j a va 2 s .c om*/ LOG.info("Using initializer class: " + initializerClass); DataSourceDescriptor dsd; if (!genSplitsInAM) { dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, remoteStagingDir, true); } else { if (initializerClass == null) { dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).build(); } else { InputInitializerDescriptor iid = InputInitializerDescriptor.create(inputInitializerClazz.getName()); dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class) .setCustomInitializerDescriptor(iid).build(); } } Vertex stage1Vertex = Vertex.create("map", ProcessorDescriptor.create(MapProcessor.class.getName()).setUserPayload(stage1Payload), dsd.getNumberOfShards(), Resource.newInstance(256, 1)); stage1Vertex.addDataSource("MRInput", dsd); Vertex stage2Vertex = Vertex.create("ireduce", ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage2Payload), 1, Resource.newInstance(256, 1)); Vertex stage3Vertex = Vertex.create("reduce", ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage3Payload), 1, Resource.newInstance(256, 1)); stage3Conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true); DataSinkDescriptor dataSinkDescriptor = MROutputLegacy .createConfigBuilder(stage3Conf, NullOutputFormat.class).build(); Assert.assertFalse(dataSinkDescriptor.getOutputDescriptor().getHistoryText().isEmpty()); stage3Vertex.addDataSink("MROutput", dataSinkDescriptor); // TODO env, resources dag.addVertex(stage1Vertex); dag.addVertex(stage2Vertex); dag.addVertex(stage3Vertex); Edge edge1 = Edge.create(stage1Vertex, stage2Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload), InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload))); Edge edge2 = Edge.create(stage2Vertex, stage3Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload), InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload))); dag.addEdge(edge1); dag.addEdge(edge2); TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); DAGClient dagClient = null; boolean reuseSession = reUseTezSession != null; TezClient tezSession = null; if (!dagViaRPC) { Preconditions.checkArgument(reuseSession == false); } if (!reuseSession) { TezConfiguration tempTezconf = new TezConfiguration(tezConf); if (!dagViaRPC) { tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); } else { tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); } tezSession = TezClient.create("testsession", tempTezconf); tezSession.start(); } else { tezSession = reUseTezSession; } if (!dagViaRPC) { // TODO Use utility method post TEZ-205 to figure out AM arguments etc. dagClient = tezSession.submitDAG(dag); } if (dagViaRPC && closeSessionBeforeSubmit) { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); boolean sentKillSession = false; while (true) { Thread.sleep(500l); ApplicationReport appReport = yarnClient .getApplicationReport(tezSession.getAppMasterApplicationId()); if (appReport == null) { continue; } YarnApplicationState appState = appReport.getYarnApplicationState(); if (!sentKillSession) { if (appState == YarnApplicationState.RUNNING) { tezSession.stop(); sentKillSession = true; } } else { if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) { LOG.info("Application completed after sending session shutdown" + ", yarnApplicationState=" + appState + ", finalAppStatus=" + appReport.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, appState); Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus()); break; } } } yarnClient.stop(); return null; } if (dagViaRPC) { LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId() + " and Dag Name=" + dag.getName()); if (additionalLocalResources != null) { tezSession.addAppMasterLocalFiles(additionalLocalResources); } dagClient = tezSession.submitDAG(dag); Assert.assertEquals(TezAppMasterStatus.RUNNING, tezSession.getAppMasterStatus()); } DAGStatus dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { LOG.info( "Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState()); Thread.sleep(500l); if (killDagWhileRunning && dagStatus.getState() == DAGStatus.State.RUNNING) { LOG.info("Killing running dag/session"); if (dagViaRPC) { tezSession.stop(); } else { dagClient.tryKillDAG(); } } dagStatus = dagClient.getDAGStatus(null); } if (!reuseSession) { tezSession.stop(); } return dagStatus.getState(); }
From source file:org.apache.tez.test.MiniTezCluster.java
License:Apache License
private void waitForAppsToFinish() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(getConfig());/*from w w w . j a va2s. c o m*/ yarnClient.start(); try { while (true) { List<ApplicationReport> appReports = yarnClient.getApplications(); Collection<ApplicationReport> unCompletedApps = Collections2.filter(appReports, new Predicate<ApplicationReport>() { @Override public boolean apply(ApplicationReport appReport) { return EnumSet .of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) .contains(appReport.getYarnApplicationState()); } }); if (unCompletedApps.size() == 0) { break; } LOG.info("wait for applications to finish in MiniTezCluster"); Thread.sleep(1000); } } catch (Exception e) { e.printStackTrace(); } finally { yarnClient.stop(); } }
From source file:org.apache.tez.test.TestExceptionPropagation.java
License:Apache License
/** * verify the diagnostics in {@link DAGStatus} is correct in non-session mode, * and also verify that diagnostics from {@link DAGStatus} should match that * from {@link ApplicationReport}//from w w w . j av a 2 s . c om * * @throws Exception */ @Test(timeout = 120000) public void testExceptionPropagationNonSession() throws Exception { try { startMiniTezCluster(); startNonSessionClient(); ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs; LOG.info("NonSession mode, Test for Exception from:" + exLocation.name()); DAG dag = createDAG(exLocation); DAGClient dagClient = tezClient.submitDAG(dag); DAGStatus dagStatus = dagClient.waitForCompletion(); String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ","); LOG.info("Diagnostics:" + diagnostics); assertTrue(diagnostics.contains(exLocation.name())); // wait for app complete (unregisterApplicationMaster is done) ApplicationId appId = tezClient.getAppMasterApplicationId(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(tezConf); yarnClient.start(); Set<YarnApplicationState> FINAL_APPLICATION_STATES = EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED); ApplicationReport appReport = null; while (true) { appReport = yarnClient.getApplicationReport(appId); Thread.sleep(1000); LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus()); LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics()); if (FINAL_APPLICATION_STATES.contains(appReport.getYarnApplicationState())) { break; } } // wait for 1 second and call getApplicationReport again to ensure get the // diagnostics // TODO remove it after YARN-2560 Thread.sleep(1000); appReport = yarnClient.getApplicationReport(appId); LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus()); LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics()); assertTrue(appReport.getDiagnostics().contains(exLocation.name())); // use "\n" as separator, because we also use it in Tez internally when // assembling the application diagnostics. assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(), appReport.getDiagnostics().trim()); } finally { stopNonSessionClient(); Thread.sleep(10 * 1000); stopTezMiniCluster(); } }
From source file:org.apache.tez.test.TestTezJobs.java
License:Apache License
@Test(timeout = 60000) public void testSimpleSessionExample() throws Exception { Path stagingDirPath = new Path("/tmp/owc-staging-dir"); remoteFs.mkdirs(stagingDirPath);/*from www . j ava2 s .c om*/ int numIterations = 2; String[] inputPaths = new String[numIterations]; String[] outputPaths = new String[numIterations]; Path[] outputDirs = new Path[numIterations]; for (int i = 0; i < numIterations; ++i) { String inputDirStr = "/tmp/owc-input-" + i + "/"; inputPaths[i] = inputDirStr; Path inputDir = new Path(inputDirStr); remoteFs.mkdirs(inputDir); generateOrderedWordCountInput(inputDir, remoteFs); String outputDirStr = "/tmp/owc-output-" + i + "/"; outputPaths[i] = outputDirStr; Path outputDir = new Path(outputDirStr); outputDirs[i] = outputDir; } TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString()); YarnClient yarnClient = YarnClient.createYarnClient(); try { yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); List<ApplicationReport> apps = yarnClient.getApplications(); int appsBeforeCount = apps != null ? apps.size() : 0; SimpleSessionExample job = new SimpleSessionExample(); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); Assert.assertTrue("SimpleSessionExample failed", job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null) == 0); for (int i = 0; i < numIterations; ++i) { verifyOutput(outputDirs[i], remoteFs); } apps = yarnClient.getApplications(); int appsAfterCount = apps != null ? apps.size() : 0; // Running in session mode. So should only create 1 more app. Assert.assertEquals(appsBeforeCount + 1, appsAfterCount); } finally { remoteFs.delete(stagingDirPath, true); if (yarnClient != null) { yarnClient.stop(); } } }
From source file:org.apache.tez.test.TestTezJobs.java
License:Apache License
@Test(timeout = 60000) public void testInvalidQueueSubmission() throws Exception { TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); YarnClient yarnClient = YarnClient.createYarnClient(); try {// w w w . j a v a2s.c om yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); SimpleSessionExample job = new SimpleSessionExample(); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent"); String[] inputPaths = new String[1]; String[] outputPaths = new String[1]; String inputDirStr = "/tmp/owc-input"; inputPaths[0] = inputDirStr; Path inputDir = new Path(inputDirStr); remoteFs.mkdirs(inputDir); String outputDirStr = "/tmp/owc-output"; outputPaths[0] = outputDirStr; job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), StringUtils.join(",", outputPaths), "2" }, null); fail("Job submission should have thrown an exception"); } catch (IOException e) { Assert.assertTrue(e.getMessage().contains("Failed to submit application to YARN")); } finally { if (yarnClient != null) { yarnClient.stop(); } } }
From source file:org.apache.tez.tests.MiniTezClusterWithTimeline.java
License:Apache License
private void waitForAppsToFinish() { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(getConfig());//from ww w . j ava2 s . c om yarnClient.start(); try { while (true) { List<ApplicationReport> appReports = yarnClient.getApplications(); Collection<ApplicationReport> unCompletedApps = Collections2.filter(appReports, new Predicate<ApplicationReport>() { @Override public boolean apply(ApplicationReport appReport) { return EnumSet .of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) .contains(appReport.getYarnApplicationState()); } }); if (unCompletedApps.size() == 0) { break; } LOG.info("wait for applications to finish in MiniTezClusterWithTimeline"); Thread.sleep(1000); } } catch (Exception e) { e.printStackTrace(); } finally { yarnClient.stop(); } }