List of usage examples for org.apache.hadoop.yarn.api.records YarnApplicationState KILLED
YarnApplicationState KILLED
To view the source code for org.apache.hadoop.yarn.api.records YarnApplicationState KILLED.
Click Source Link
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.client.Client.java
License:Apache License
/** * TODO: consider the scenarios where we dont get enough containers * - we need to re-submit the job till we get the containers alloc'd * //from www . ja va2 s.com */ @Override public int run(String[] args) throws Exception { //System.out.println("IR: Client.run() [start]"); if (args.length < 1) LOG.info("No configuration file specified, using default (" + ConfigFields.DEFAULT_CONFIG_FILE + ")"); long startTime = System.currentTimeMillis(); String configFile = (args.length < 1) ? ConfigFields.DEFAULT_CONFIG_FILE : args[0]; Properties props = new Properties(); Configuration conf = getConf(); try { FileInputStream fis = new FileInputStream(configFile); props.load(fis); } catch (FileNotFoundException ex) { throw ex; // TODO: be nice } catch (IOException ex) { throw ex; // TODO: be nice } // Make sure we have some bare minimums ConfigFields.validateConfig(props); if (LOG.isDebugEnabled()) { LOG.debug("Loaded configuration: "); for (Map.Entry<Object, Object> entry : props.entrySet()) { LOG.debug(entry.getKey() + "=" + entry.getValue()); } } // TODO: make sure input file(s), libs, etc. actually exist! // Ensure our input path exists Path p = new Path(props.getProperty(ConfigFields.APP_INPUT_PATH)); FileSystem fs = FileSystem.get(conf); if (!fs.exists(p)) throw new FileNotFoundException("Input path not found: " + p.toString() + " (in " + fs.getUri() + ")"); LOG.info("Using input path: " + p.toString()); // Connect ResourceManagerHandler rmHandler = new ResourceManagerHandler(conf, null); rmHandler.getClientResourceManager(); // Create an Application request/ID ApplicationId appId = rmHandler.getApplicationId(); // Our AppId String appName = props.getProperty(ConfigFields.APP_NAME, ConfigFields.DEFAULT_APP_NAME).replace(' ', '_'); LOG.info("Got an application, id=" + appId + ", appName=" + appName); // Copy resources to [HD]FS LOG.debug("Copying resources to filesystem"); Utils.copyLocalResourcesToFs(props, conf, appId, appName); // Local resources Utils.copyLocalResourceToFs(configFile, ConfigFields.APP_CONFIG_FILE, conf, appId, appName); // Config file try { Utils.copyLocalResourceToFs("log4j.properties", "log4j.properties", conf, appId, appName); // Log4j } catch (FileNotFoundException ex) { LOG.warn("log4j.properties file not found"); } // Create our context List<String> commands = Utils.getMasterCommand(conf, props); Map<String, LocalResource> localResources = Utils.getLocalResourcesForApplication(conf, appId, appName, props, LocalResourceVisibility.APPLICATION); // Submit app rmHandler.submitApplication(appId, appName, Utils.getEnvironment(conf, props), localResources, commands, Integer.parseInt(props.getProperty(ConfigFields.YARN_MEMORY, "512"))); /* * TODO: * - look at updating this code region to make sure job is submitted! * */ StopWatch watch = new StopWatch(); watch.start(); // Wait for app to complete while (true) { Thread.sleep(2000); ApplicationReport report = rmHandler.getApplicationReport(appId); LOG.info("IterativeReduce report: " + " appId=" + appId.getId() + ", state: " + report.getYarnApplicationState().toString() + ", Running Time: " + watch.toString()); //report.getDiagnostics() if (YarnApplicationState.FINISHED == report.getYarnApplicationState()) { LOG.info("Application finished in " + (System.currentTimeMillis() - startTime) + "ms"); if (FinalApplicationStatus.SUCCEEDED == report.getFinalApplicationStatus()) { LOG.info("Application completed succesfully."); return 0; } else { LOG.info("Application completed with en error: " + report.getDiagnostics()); return -1; } } else if (YarnApplicationState.FAILED == report.getYarnApplicationState() || YarnApplicationState.KILLED == report.getYarnApplicationState()) { LOG.info("Application completed with a failed or killed state: " + report.getDiagnostics()); return -1; } } }
From source file:org.dknight.app.UnmanagedAMLauncher.java
License:Apache License
public boolean run() throws IOException, YarnException { LOG.info("Starting Client"); // Connect to ResourceManager rmClient.start();// w w w .j a v a2 s .com try { // Create launch context for app master LOG.info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = rmClient.createApplication() .getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName(appName); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); // unmanaged AM appContext.setUnmanagedAM(true); LOG.info("Setting unmanaged AM"); // Submit the application to the applications manager LOG.info("Submitting application to ASM"); rmClient.submitApplication(appContext); // Monitor the application to wait for launch state ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching application with id: " + attemptId); // launch AM launchAM(attemptId); // Monitor the application for end state appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED)); YarnApplicationState appState = appReport.getYarnApplicationState(); FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus(); LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus); boolean success; if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) { LOG.info("Application has completed successfully."); success = true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString()); success = false; } return success; } finally { rmClient.stop(); } }
From source file:org.elasticsearch.hadoop.yarn.client.ClientRpc.java
License:Apache License
public void waitForApp(ApplicationId appId, long timeout) { boolean repeat = false; long start = System.currentTimeMillis(); do {//from w w w . j a v a 2 s .co m try { ApplicationReport appReport = client.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); repeat = (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED); if (repeat) { Thread.sleep(500); } } catch (Exception ex) { throw new EsYarnException(ex); } } while (repeat && (System.currentTimeMillis() - start) < timeout); }
From source file:org.hdl.caffe.yarn.app.Client.java
License:Apache License
/** * Monitor the submitted application for completion. * * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException/* w w w . j a v a2s . c om*/ * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", caffeAppFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus caffeStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.RUNNING == state) { if (appRpc == null) { String hostname = report.getHost(); int port = report.getRpcPort(); LOG.info("Application master rpc host: " + hostname + "; port: " + port); appRpc = new CaffeApplicationRpcClient(hostname, port).getRpc(); } } if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == caffeStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", appFinalState=" + caffeStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", appFinalState=" + caffeStatus.toString() + ". Breaking monitoring loop"); return false; } } }
From source file:org.hdl.tensorflow.yarn.client.LaunchCluster.java
License:Apache License
boolean awaitApplication(ApplicationId appId) throws Exception { Set<YarnApplicationState> terminated = Sets.newHashSet(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED); while (true) { ApplicationReport report = yarnClient.getApplicationReport(appId); YarnApplicationState state = report.getYarnApplicationState(); if (state.equals(YarnApplicationState.RUNNING)) { ClusterSpec clusterSpec = Client.getClusterSpec(yarnClient, appId); if (isClusterSpecSatisfied(clusterSpec)) { System.out.println("ClusterSpec: " + Utils.toJsonString(clusterSpec.getCluster())); return true; }/*from ww w. j a v a 2 s .c om*/ } else if (terminated.contains(state)) { return false; } else { Thread.sleep(1000); } } }
From source file:org.hortonworks.dovetail.client.Client.java
License:Apache License
/** * Monitor the submitted application for completion. Kill application if * time expires.//w w w .ja va 2 s. c om * * @param appId * Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.finest("Thread sleep in monitoring loop interrupted"); } ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dovetailStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dovetailStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DovetailFinalStatus=" + dovetailStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DovetailFinalStatus=" + dovetailStatus.toString() + ". Breaking monitoring loop"); return false; } } }
From source file:org.springframework.yarn.boot.app.YarnKillApplication.java
License:Apache License
public String run(String... args) { SpringApplicationBuilder builder = new SpringApplicationBuilder(); builder.web(false);//w ww . j ava2 s . co m builder.sources(YarnKillApplication.class, OperationProperties.class); SpringYarnBootUtils.addSources(builder, sources.toArray(new Object[0])); SpringYarnBootUtils.addProfiles(builder, profiles.toArray(new String[0])); if (StringUtils.hasText(applicationBaseDir)) { appProperties.setProperty("spring.yarn.applicationDir", applicationBaseDir + applicationVersion + "/"); } SpringYarnBootUtils.addApplicationListener(builder, appProperties); SpringApplicationTemplate template = new SpringApplicationTemplate(builder); return template.execute(new SpringApplicationCallback<String>() { @Override public String runWithSpringApplication(ApplicationContext context) throws Exception { YarnClient client = context.getBean(YarnClient.class); OperationProperties operationProperties = context.getBean(OperationProperties.class); ApplicationId applicationId = ConverterUtils .toApplicationId(operationProperties.getApplicationId()); ApplicationReport report = client.getApplicationReport(applicationId); if (report.getYarnApplicationState() == YarnApplicationState.FINISHED || report.getYarnApplicationState() == YarnApplicationState.KILLED || report.getYarnApplicationState() == YarnApplicationState.FAILED) { return "Application " + applicationId + " is not running"; } else { client.killApplication(applicationId); return "Kill request for " + applicationId + " sent"; } } }, args); }
From source file:org.springframework.yarn.examples.ComplexExampleTests.java
License:Apache License
@Test @Timed(millis = 360000)/* w w w. j av a2 s . co m*/ public void testAppSubmission() throws Exception { // submit and wait running state ApplicationId applicationId = submitApplication(); assertNotNull(applicationId); YarnApplicationState state = waitState(applicationId, 120, TimeUnit.SECONDS, YarnApplicationState.RUNNING); assertNotNull(state); assertTrue(state.equals(YarnApplicationState.RUNNING)); // check registered rpc port ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); String rpcHost = applicationReport.getHost(); int rpcPort = applicationReport.getRpcPort(); assertThat(rpcHost, notNullValue()); assertThat(rpcPort, greaterThan(0)); File baseDir = getYarnCluster().getYarnWorkDir(); String baseUrl = findXdBaseUrl(applicationId); // we're starting for zero, launch and wait setContainerCountViaThrift(1, "default", rpcHost, rpcPort); setContainerCountViaThrift(1, "xdgroup", rpcHost, rpcPort); int count = ApplicationTestUtils.waitResourcesMatchCount(baseDir, PAT_C_STDOUT, 2, true, null); assertThat(count, is(2)); // do ticktock request and wait logging message StreamDefinitionResource stream = createTickTockStream(baseUrl); // for some reason stream name is null, bug in xd? //assertThat(stream.getName(), is("ticktock")); count = ApplicationTestUtils.waitResourcesMatchCount(baseDir, PAT_C_STDOUT, 1, true, "LoggingHandler"); assertThat(count, is(1)); // resize and wait setContainerCountViaThrift(2, "default", rpcHost, rpcPort); setContainerCountViaThrift(2, "xdgroup", rpcHost, rpcPort); count = ApplicationTestUtils.waitResourcesMatchCount(baseDir, PAT_C_STDOUT, 4, true, null); assertThat(count, is(4)); deleteTickTockStream(baseUrl); // long running app, kill it and check that state is KILLED killApplication(applicationId); state = waitState(applicationId, 30, TimeUnit.SECONDS, YarnApplicationState.KILLED); assertThat(state, is(YarnApplicationState.KILLED)); // appmaster and 4 container should make it 10 log files Resource[] resources = ApplicationTestUtils.matchResources(baseDir, PAT_ALL); assertThat(resources, notNullValue()); assertThat(resources.length, is(10)); }
From source file:org.springframework.yarn.examples.KillApplicationTests.java
License:Apache License
@Test @Timed(millis = 150000)// w w w . j a va 2s.c o m public void testAppSubmission() throws Exception { ApplicationId applicationId = submitApplication(); YarnApplicationState state = waitState(applicationId, 120, TimeUnit.SECONDS, YarnApplicationState.RUNNING); assertThat(state, notNullValue()); getYarnClient().killApplication(applicationId); state = waitState(applicationId, 20, TimeUnit.SECONDS, YarnApplicationState.KILLED); assertThat(state, notNullValue()); assertThat(state, is(YarnApplicationState.KILLED)); }
From source file:org.springframework.yarn.examples.SimpleExampleTests.java
License:Apache License
@Test @Timed(millis = 240000)/*w ww.jav a2 s . c o m*/ public void testAppSubmission() throws Exception { // submit and wait running state ApplicationId applicationId = submitApplication(); assertNotNull(applicationId); YarnApplicationState state = waitState(applicationId, 120, TimeUnit.SECONDS, YarnApplicationState.RUNNING); assertNotNull(state); assertTrue(state.equals(YarnApplicationState.RUNNING)); // wait and do ticktock put Thread.sleep(20000); doTickTockTimeLogPut(applicationId); // assertTrue("Ticktock request failed", doTickTockTimeLogPut(applicationId)); // wait a bit for spring-xd containers to log something Thread.sleep(10000); // long running app, kill it and check that state is KILLED killApplication(applicationId); state = getState(applicationId); assertTrue(state.equals(YarnApplicationState.KILLED)); // get log files File workDir = getYarnCluster().getYarnWorkDir(); PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); String locationPattern = "file:" + workDir.getAbsolutePath() + "/**/*.std*"; Resource[] resources = resolver.getResources(locationPattern); // appmaster and 1 container should make it 4 log files assertThat(resources, notNullValue()); assertThat(resources.length, is(4)); // do some checks for log file content for (Resource res : resources) { File file = res.getFile(); if (file.getName().endsWith("stdout")) { // there has to be some content in stdout file assertThat(file.length(), greaterThan(0l)); if (file.getName().equals("Container.stdout")) { Scanner scanner = new Scanner(file); String content = scanner.useDelimiter("\\A").next(); scanner.close(); // this is what xd container should log assertThat(content, containsString("LoggingHandler")); } } } }