List of usage examples for org.apache.hadoop.yarn.api.records YarnApplicationState KILLED
YarnApplicationState KILLED
To view the source code for org.apache.hadoop.yarn.api.records YarnApplicationState KILLED.
Click Source Link
From source file:org.apache.flink.yarn.YarnClusterClient.java
License:Apache License
/** * Shuts down the Yarn application// ww w .ja va 2s. co m */ public void shutdownCluster() { if (hasBeenShutDown.getAndSet(true)) { return; } if (!isConnected) { throw new IllegalStateException("The cluster has been not been connected to the ApplicationMaster."); } try { Runtime.getRuntime().removeShutdownHook(clientShutdownHook); } catch (IllegalStateException e) { // we are already in the shutdown hook } LOG.info("Sending shutdown request to the Application Master"); try { Future<Object> response = Patterns.ask(applicationClient.get(), new YarnMessages.LocalStopYarnSession( getApplicationStatus(), "Flink YARN Client requested shutdown"), new Timeout(akkaDuration)); Await.ready(response, akkaDuration); } catch (Exception e) { LOG.warn("Error while stopping YARN cluster.", e); } try { File propertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(flinkConfig); if (propertiesFile.isFile()) { if (propertiesFile.delete()) { LOG.info("Deleted Yarn properties file at {}", propertiesFile.getAbsoluteFile().toString()); } else { LOG.warn("Couldn't delete Yarn properties file at {}", propertiesFile.getAbsoluteFile().toString()); } } } catch (Exception e) { LOG.warn("Exception while deleting the JobManager address file", e); } if (sessionFilesDir != null) { LOG.info("Deleting files in " + sessionFilesDir); try { FileSystem shutFS = FileSystem.get(hadoopConfig); shutFS.delete(sessionFilesDir, true); // delete conf and jar file. shutFS.close(); } catch (IOException e) { LOG.error("Could not delete the Flink jar and configuration files in HDFS..", e); } } else { LOG.warn("Session file directory not set. Not deleting session files"); } try { pollingRunner.stopRunner(); pollingRunner.join(1000); } catch (InterruptedException e) { LOG.warn("Shutdown of the polling runner was interrupted", e); Thread.currentThread().interrupt(); } try { ApplicationReport appReport = yarnClient.getApplicationReport(appId); LOG.info("Application " + appId + " finished with state " + appReport.getYarnApplicationState() + " and final state " + appReport.getFinalApplicationStatus() + " at " + appReport.getFinishTime()); if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED) { LOG.warn("Application failed. Diagnostics " + appReport.getDiagnostics()); LOG.warn("If log aggregation is activated in the Hadoop cluster, we recommend to retrieve " + "the full application log using this command:" + System.lineSeparator() + "\tyarn logs -applicationId " + appReport.getApplicationId() + System.lineSeparator() + "(It sometimes takes a few seconds until the logs are aggregated)"); } } catch (Exception e) { LOG.warn("Couldn't get final report", e); } LOG.info("YARN Client is shutting down"); yarnClient.stop(); // actorRunner is using the yarnClient. yarnClient = null; // set null to clearly see if somebody wants to access it afterwards. }
From source file:org.apache.flink.yarn.YARNSessionFIFOITCase.java
License:Apache License
/** * Test regular operation, including command line parameter parsing. *///from ww w . j a va2 s .c o m @Test(timeout = 60000) // timeout after a minute. public void testDetachedMode() { LOG.info("Starting testDetachedMode()"); addTestAppender(FlinkYarnSessionCli.class, Level.INFO); Runner runner = startWithArgs(new String[] { "-j", flinkUberjar.getAbsolutePath(), "-t", flinkLibFolder.getAbsolutePath(), "-n", "1", "-jm", "768", "-tm", "1024", "--name", "MyCustomName", // test setting a custom name "--detached" }, "Flink JobManager is now running on", RunTypes.YARN_SESSION); checkForLogString("The Flink YARN client has been started in detached mode"); Assert.assertFalse("The runner should detach.", runner.isAlive()); LOG.info("Waiting until two containers are running"); // wait until two containers are running while (getRunningContainers() < 2) { sleep(500); } LOG.info("Two containers are running. Killing the application"); // kill application "externally". try { YarnClient yc = YarnClient.createYarnClient(); yc.init(yarnConfiguration); yc.start(); List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING)); Assert.assertEquals(1, apps.size()); // Only one running ApplicationReport app = apps.get(0); Assert.assertEquals("MyCustomName", app.getName()); ApplicationId id = app.getApplicationId(); yc.killApplication(id); while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0) { sleep(500); } } catch (Throwable t) { LOG.warn("Killing failed", t); Assert.fail(); } LOG.info("Finished testDetachedMode()"); }
From source file:org.apache.flink.yarn.YarnTestBase.java
License:Apache License
@Before public void checkClusterEmpty() throws IOException, YarnException { if (yarnClient == null) { yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfiguration); yarnClient.start();//from w ww. j a v a2 s. c o m } List<ApplicationReport> apps = yarnClient.getApplications(); for (ApplicationReport app : apps) { if (app.getYarnApplicationState() != YarnApplicationState.FINISHED && app.getYarnApplicationState() != YarnApplicationState.KILLED && app.getYarnApplicationState() != YarnApplicationState.FAILED) { Assert.fail("There is at least one application on the cluster is not finished." + "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState()); } } }
From source file:org.apache.giraph.yarn.GiraphYarnClient.java
License:Apache License
/** * Assess whether job is already finished/failed and 'done' flag needs to be * set, prints progress display for client if all is going well. * @param report the application report to assess. * @return true if job report indicates the job run is over. */// w ww . jav a 2 s . c o m private boolean checkProgress(final ApplicationReport report) { YarnApplicationState jobState = report.getYarnApplicationState(); if (jobState == YarnApplicationState.FINISHED || jobState == YarnApplicationState.KILLED) { return true; } else if (jobState == YarnApplicationState.FAILED) { LOG.error(jobName + " reports FAILED state, diagnostics show: " + report.getDiagnostics()); return true; } else { if (reportCounter++ % 5 == 0) { displayJobReport(report); } } return false; }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncherTest.java
License:Apache License
@Test(enabled = false, groups = { "disabledOnTravis" }, dependsOnMethods = "testSetupAndSubmitApplication")
public void testGetReconnectableApplicationId() throws Exception {
Assert.assertEquals(this.gobblinYarnAppLauncher.getReconnectableApplicationId().get(), this.applicationId);
this.yarnClient.killApplication(this.applicationId);
Assert.assertEquals(yarnClient.getApplicationReport(applicationId).getYarnApplicationState(),
YarnApplicationState.KILLED, "Application not killed");
// takes some time for kill to take effect and app master to go down
Thread.sleep(5000);//from w w w . j a v a 2 s . c o m
}
From source file:org.apache.hama.bsp.YARNBSPJobClient.java
License:Apache License
private boolean monitorApplication(ApplicationId appId) throws IOException, YarnException { while (true) { try {/*from w w w . j av a2 s. c o m*/ Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
/** * @return true if successfully completed, it will print status every X seconds *///from w ww . j a v a 2 s . c om public boolean waitUntilDone() { String prevReport = ""; HelixConnection connection = null; while (true) { try { // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(_appId); String reportMessage = generateReport(report); if (!reportMessage.equals(prevReport)) { LOG.info(reportMessage); } YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (YarnApplicationState.RUNNING == state) { if (connection == null) { String hostName = null; int ind = report.getHost().indexOf('/'); if (ind > -1) { hostName = report.getHost().substring(ind + 1); } else { hostName = report.getHost(); } connection = new ZkHelixConnection(hostName + ":2181"); try { connection.connect(); } catch (Exception e) { LOG.warn("AppMaster started but not yet initialized"); connection = null; } } if (connection.isConnected()) { AppStatusReportGenerator generator = new AppStatusReportGenerator(); ClusterId clusterId = ClusterId.from(_applicationSpec.getAppName()); String generateReport = generator.generateReport(connection, clusterId); LOG.info(generateReport); } } prevReport = reportMessage; Thread.sleep(10000); } catch (Exception e) { LOG.error("Exception while getting info", e); break; } } return true; }
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
/** * Monitor the submitted application for completion. * Kill application if time expires./* w ww .j a v a2s . com*/ * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.RUNNING == state) { LOG.info("Application is running..."); return true; } if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * Implement the list action./*from w w w . j a v a 2 s . c o m*/ * @param clustername List out specific instance name * @param args Action list arguments * @return 0 if one or more entries were listed * @throws IOException * @throws YarnException * @throws UnknownApplicationInstanceException if a specific instance * was named but it was not found */ @Override @VisibleForTesting public int actionList(String clustername, ActionListArgs args) throws IOException, YarnException { verifyBindingsDefined(); boolean live = args.live; String state = args.state; boolean verbose = args.verbose; if (live && !state.isEmpty()) { throw new BadCommandArgumentsException( Arguments.ARG_LIVE + " and " + Arguments.ARG_STATE + " are exclusive"); } // flag to indicate only services in a specific state are to be listed boolean listOnlyInState = live || !state.isEmpty(); YarnApplicationState min, max; if (live) { min = YarnApplicationState.NEW; max = YarnApplicationState.RUNNING; } else if (!state.isEmpty()) { YarnApplicationState stateVal = extractYarnApplicationState(state); min = max = stateVal; } else { min = YarnApplicationState.NEW; max = YarnApplicationState.KILLED; } // get the complete list of persistent instances Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances(); if (persistentInstances.isEmpty() && isUnset(clustername)) { // an empty listing is a success if no cluster was named log.debug("No application instances found"); return EXIT_SUCCESS; } // and those the RM knows about List<ApplicationReport> instances = listSliderInstances(null); SliderUtils.sortApplicationsByMostRecent(instances); Map<String, ApplicationReport> reportMap = SliderUtils.buildApplicationReportMap(instances, min, max); log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", persistentInstances.size(), instances.size(), min, max, reportMap.size()); if (isSet(clustername)) { // only one instance is expected // resolve the persistent value Path persistent = persistentInstances.get(clustername); if (persistent == null) { throw unknownClusterException(clustername); } // create a new map with only that instance in it. // this restricts the output of results to this instance persistentInstances = new HashMap<String, Path>(); persistentInstances.put(clustername, persistent); } // at this point there is either the entire list or a stripped down instance int listed = 0; for (String name : persistentInstances.keySet()) { ApplicationReport report = reportMap.get(name); if (!listOnlyInState || report != null) { // list the details if all were requested, or the filtering contained // a report listed++; String details = instanceDetailsToString(name, report, verbose); print(details); } } return listed > 0 ? EXIT_SUCCESS : EXIT_FALSE; }
From source file:org.apache.slider.common.tools.TestSliderUtils.java
License:Apache License
@Test public void testApplicationReportComparison() { List<ApplicationReport> instances = getApplicationReports(); SliderUtils.sortApplicationsByMostRecent(instances); Assert.assertEquals(1000, instances.get(0).getStartTime()); Assert.assertEquals(1000, instances.get(1).getStartTime()); Assert.assertEquals(1000, instances.get(2).getStartTime()); Assert.assertEquals(1000, instances.get(3).getStartTime()); instances = getApplicationReports(); SliderUtils.sortApplicationReport(instances); Assert.assertEquals(1000, instances.get(0).getStartTime()); Assert.assertEquals(1000, instances.get(1).getStartTime()); Assert.assertEquals(1000, instances.get(2).getStartTime()); Assert.assertEquals(1000, instances.get(3).getStartTime()); Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED || instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING); Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED || instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING); Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED || instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING); Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED); }