List of usage examples for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED
FinalApplicationStatus SUCCEEDED
To view the source code for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED.
Click Source Link
From source file:org.hortonworks.dovetail.am.AppMaster.java
License:Apache License
private void finish() { for (Thread launchThread : launchThreads) { try {// www.jav a 2s.c o m launchThread.join(10000); } catch (InterruptedException e) { LOG.info("Exception thrown in thread join: " + e.getMessage()); e.printStackTrace(); } } LOG.info("Application completed. Stopping running containers"); nmClientAsync.stop(); LOG.info("Application completed. Signalling finish to RM"); FinalApplicationStatus appStatus; String appMessage = null; success = true; if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numContainers) { appStatus = FinalApplicationStatus.SUCCEEDED; } else { appStatus = FinalApplicationStatus.FAILED; appMessage = "Diagnostics." + ", total=" + numContainers + ", completed=" + numCompletedContainers.get() + ", allocated=" + numAllocatedContainers.get() + ", failed=" + numFailedContainers.get(); success = false; } try { resourceManager.unregisterApplicationMaster(appStatus, appMessage, null); } catch (YarnException ex) { LOG.log(Level.SEVERE, "Failed to unregister application", ex); } catch (IOException e) { LOG.log(Level.SEVERE, "Failed to unregister application", e); } done = true; resourceManager.stop(); }
From source file:org.hortonworks.dovetail.client.Client.java
License:Apache License
/** * Monitor the submitted application for completion. Kill application if * time expires./*from w w w. j av a2 s. co m*/ * * @param appId * Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.finest("Thread sleep in monitoring loop interrupted"); } ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dovetailStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dovetailStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DovetailFinalStatus=" + dovetailStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DovetailFinalStatus=" + dovetailStatus.toString() + ". Breaking monitoring loop"); return false; } } }
From source file:org.springframework.yarn.am.AbstractAppmaster.java
License:Apache License
/** * Finish appmaster by sending request to resource manager. Default * application status is {@code FinalApplicationStatus.SUCCEEDED} which * can be changed using method {@link #setFinalApplicationStatus(FinalApplicationStatus)}. * * @return the finish application master response *///from ww w. ja v a2 s . c o m protected FinishApplicationMasterResponse finishAppmaster() { boolean clean = getResourceLocalizer().clean(); log.info("Status of resource localizer clean operation is " + clean); // starting from 2.1.x applicationAttemptId is part of the token and // doesn't exist in finish request. We still keep it around as per // old concept. Assert.notNull(applicationAttemptId, "applicationAttemptId must be set"); if (!applicationRegistered) { log.warn("Not sending finish request because we're not registered"); return null; } FinishApplicationMasterRequest finishReq = Records.newRecord(FinishApplicationMasterRequest.class); // assume succeed if not set FinalApplicationStatus status = finalApplicationStatus != null ? finalApplicationStatus : FinalApplicationStatus.SUCCEEDED; if (log.isDebugEnabled()) { log.debug("Sending finish request to resource manager. Current applicationAttemptId=" + applicationAttemptId + " with status=" + status); } finishReq.setFinalApplicationStatus(status); return rmTemplate.finish(finishReq); }
From source file:org.starschema.hadoop.yarn.applications.distributedshell.ApplicationMaster.java
License:Apache License
@VisibleForTesting protected boolean finish() { // wait for completion. while (!done && (numCompletedContainers.get() < numTotalContainers)) { try {/*www . ja v a2 s. c o m*/ Thread.sleep(200); } catch (InterruptedException ex) { } } if (timelineClient != null) { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi); } // Join all launched threads // needed for when we time out // and we need to release containers for (Thread launchThread : launchThreads) { try { launchThread.join(10000); } catch (InterruptedException e) { LOG.info("Exception thrown in thread join: " + e.getMessage()); e.printStackTrace(); } } // When the application completes, it should stop all running containers LOG.info("Application completed. Stopping running containers"); nmClientAsync.stop(); // When the application completes, it should send a finish application // signal to the RM LOG.info("Application completed. Signalling finish to RM"); FinalApplicationStatus appStatus; String appMessage = null; boolean success = true; if (numFailedContainers.get() == 0 && numCompletedContainers.get() >= numTotalContainers) { appStatus = FinalApplicationStatus.SUCCEEDED; } else { appStatus = FinalApplicationStatus.FAILED; appMessage = "Diagnostics." + ", total=" + numTotalContainers + ", completed=" + numCompletedContainers.get() + ", allocated=" + numAllocatedContainers.get() + ", failed=" + numFailedContainers.get(); LOG.info(appMessage); success = false; } try { amRMClient.unregisterApplicationMaster(appStatus, appMessage, null); } catch (YarnException ex) { LOG.error("Failed to unregister application", ex); } catch (IOException e) { LOG.error("Failed to unregister application", e); } amRMClient.stop(); // Stop Timeline Client if (timelineClient != null) { timelineClient.stop(); } return success; }
From source file:org.starschema.hadoop.yarn.applications.distributedshell.Client.java
License:Apache License
/** * Monitor the submitted application for completion. * Kill application if time expires. /*from w ww . ja v a 2s . c o m*/ * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } // if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { // LOG.info("Reached client specified timeout for application. Killing application"); // forceKillApplication(appId); // return false; // } } }
From source file:oz.hadoop.yarn.api.core.ApplicationContainerLauncherImpl.java
License:Apache License
/** * //from w w w .ja va2 s. c o m */ @Override void doShutDown() throws Exception { String suffix = this.applicationSpecification.getString(YayaConstants.APPLICATION_NAME) + "_master/" + this.applicationSpecification.getInt(YayaConstants.APP_ID) + "/"; FileSystem fs = FileSystem.get(this.yarnConfig); Path dst = new Path(fs.getHomeDirectory(), suffix); fs.delete(dst, true); if (logger.isInfoEnabled()) { logger.info("Deleted application jars: " + dst.toString()); } FinalApplicationStatus status = (this.error != null) ? FinalApplicationStatus.FAILED : FinalApplicationStatus.SUCCEEDED; //this.resourceManagerClient.getClusterNodeCount() //this.resourceManagerClient.getFailureCause() logger.info("Unregistering the Application Master"); this.resourceManagerClient.unregisterApplicationMaster(status, this.generateExitMessage(status), null); logger.info("Shutting down Node Manager Client"); this.nodeManagerClient.stop(); logger.info("Shutting down Resource Manager Client"); this.resourceManagerClient.stop(); }
From source file:probos.TestProblem.java
License:Open Source License
void dotest(int N, String defn) throws Exception { if (System.getenv("HADOOP_HOME") == null && System.getenv("HADOOP_COMMON_HOME") == null) fail("HADOOP_HOME must be set"); if (System.getenv("JAVA_HOME") == null) fail("JAVA_HOME must be set"); File luaFile = File.createTempFile("kittenFile", ".lua"); FileWriter w = new FileWriter(luaFile); w.write(defn);//w w w . ja va 2 s . c o m w.close(); Map<String, Object> extraLuaValues = ImmutableMap.<String, Object>of(); Map<String, String> extraLocalResources = ImmutableMap.<String, String>of(); YarnClientParameters params = new LuaYarnClientParameters(luaFile.toString(), "probos", yConf, extraLuaValues, extraLocalResources); YarnClientService service = new YarnClientServiceImpl(params); service.startAndWait(); while (!service.isApplicationFinished()) { Thread.sleep(1000); } assertEquals(FinalApplicationStatus.SUCCEEDED, service.getFinalReport().getFinalApplicationStatus()); ApplicationAttemptId aaid = service.getFinalReport().getCurrentApplicationAttemptId(); YarnClient yc = new YarnClientFactory(this.yConf).connect(); List<ContainerReport> lcr = yc.getContainers(aaid); for (ContainerReport cr : lcr) { String stdErrURL = "http:" + cr.getLogUrl() + "/stderr?start=0"; System.err.println(cr.getContainerId().toString() + " " + stdErrURL); String stderr = getURL(stdErrURL); System.err.println(stderr); assertFalse("Container" + cr.getContainerId().toString() + " " + stderr, stderr.contains("ArrayIndexOutOfBoundsException")); } //service.getFinalReport().get System.err.println(); Thread.sleep(60000); for (int id = 1; id <= N; id++) for (String type : new String[] { "o", "e" }) { String file = HERE + "/testHostname." + type + "1-" + id; assertTrue("File not found " + file, new File(file).exists()); } }
From source file:proxyyarn.ProxyYarn.java
License:Apache License
private boolean monitorApplication(YarnClient yarnClient, ApplicationId appId) throws YarnException, IOException { long clientStartTime = System.currentTimeMillis(); while (true) { // Check app status every 1 second. try {//w w w . j av a2 s . com Thread.sleep(1000); } catch (InterruptedException e) { log.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); log.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { log.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { log.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { log.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { log.info("Reached client specified timeout for application. Killing application"); forceKillApplication(yarnClient, appId); return false; } } }
From source file:proxyyarn.ProxyYarnAppMaster.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf);/*from w ww . j a v a2 s. co m*/ amRMClient.start(); // NMCallbackHandler containerListener = createNMCallbackHandler(); // nmClientAsync = new NMClientAsyncImpl(containerListener); // nmClientAsync.init(conf); // nmClientAsync.start(); // Register self with ResourceManager // This will start heartbeating to the RM String appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, -1, ""); ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); for (int i = 0; i < 50; i++) { System.out.println("ProxyYarnAppMaster is running! Iteration " + i); Thread.sleep(1000); } // When the application completes, it should stop all running containers // log.info("Application completed. Stopping running containers"); // nmClientAsync.stop(); // When the application completes, it should send a finish application // signal to the RM log.info("Application completed. Signalling finish to RM"); // FinalApplicationStatus appStatus; String appMessage = null; success = true; // if (numFailedContainers.get() == 0 && numCompletedContainers.get() == numTotalContainers) { // appStatus = FinalApplicationStatus.SUCCEEDED; // } else { // appStatus = FinalApplicationStatus.FAILED; // appMessage = "Diagnostics." + ", total=" + numTotalContainers + ", completed=" + numCompletedContainers.get() + ", allocated=" // + numAllocatedContainers.get() + ", failed=" + numFailedContainers.get(); // success = false; // } try { amRMClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMessage, null); } catch (YarnException ex) { log.error("Failed to unregister application", ex); } catch (IOException e) { log.error("Failed to unregister application", e); } amRMClient.stop(); System.exit(0); }
From source file:runtime.starter.MPJAppMaster.java
License:Open Source License
public void run() throws Exception { try {/*w w w . j av a 2s.c om*/ appMasterSock = new Socket(serverName, ioServerPort); //redirecting stdout and stderr System.setOut(new PrintStream(appMasterSock.getOutputStream(), true)); System.setErr(new PrintStream(appMasterSock.getOutputStream(), true)); } catch (Exception exp) { exp.printStackTrace(); } FileSystem fs = FileSystem.get(conf); Path wrapperDest = new Path(wrapperPath); FileStatus destStatus = fs.getFileStatus(wrapperDest); Path userFileDest = new Path(userJarPath); FileStatus destStatusClass = fs.getFileStatus(userFileDest); // Initialize AM <--> RM communication protocol AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); // Initialize AM <--> NM communication protocol NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager RegisterApplicationMasterResponse registerResponse = rmClient.registerApplicationMaster("", 0, ""); // Priority for containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(mpjContainerPriority); maxMem = registerResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { System.out.println("[MPJAppMaster]: Max memory capability resources " + "in cluster: " + maxMem); } if (containerMem > maxMem) { System.out.println("[MPJAppMaster]: container memory specified above " + "threshold of cluster! Using maximum memory for " + "containers: " + containerMem); containerMem = maxMem; } maxCores = registerResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { System.out.println("[MPJAppMaster]: Max v-cores capability resources " + "in cluster: " + maxCores); } if (containerCores > maxCores) { System.out.println("[MPJAppMaster]: virtual cores specified above " + "threshold of cluster! Using maximum v-cores for " + "containers: " + containerCores); containerCores = maxCores; } // Resource requirements for containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMem); capability.setVirtualCores(containerCores); // Make container requests to ResourceManager for (int i = 0; i < np; ++i) { ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerReq); } Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // Creating Local Resource for Wrapper LocalResource wrapperJar = Records.newRecord(LocalResource.class); wrapperJar.setResource(ConverterUtils.getYarnUrlFromPath(wrapperDest)); wrapperJar.setSize(destStatus.getLen()); wrapperJar.setTimestamp(destStatus.getModificationTime()); wrapperJar.setType(LocalResourceType.ARCHIVE); wrapperJar.setVisibility(LocalResourceVisibility.APPLICATION); // Creating Local Resource for UserClass LocalResource userClass = Records.newRecord(LocalResource.class); userClass.setResource(ConverterUtils.getYarnUrlFromPath(userFileDest)); userClass.setSize(destStatusClass.getLen()); userClass.setTimestamp(destStatusClass.getModificationTime()); userClass.setType(LocalResourceType.ARCHIVE); userClass.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put("mpj-yarn-wrapper.jar", wrapperJar); localResources.put("user-code.jar", userClass); while (allocatedContainers < np) { AllocateResponse response = rmClient.allocate(0); mpiContainers.addAll(response.getAllocatedContainers()); allocatedContainers = mpiContainers.size(); if (allocatedContainers != np) { Thread.sleep(100); } } if (debugYarn) { System.out.println("[MPJAppMaster]: launching " + allocatedContainers + " containers"); } for (Container container : mpiContainers) { ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add(" $JAVA_HOME/bin/java"); commands.add(" -Xmx" + containerMem + "m"); commands.add(" runtime.starter.MPJYarnWrapper"); commands.add("--serverName"); commands.add(serverName); // server name commands.add("--ioServerPort"); commands.add(Integer.toString(ioServerPort)); // IO server port commands.add("--deviceName"); commands.add(deviceName); // device name commands.add("--className"); commands.add(className); // class name commands.add("--psl"); commands.add(psl); // protocol switch limit commands.add("--np"); commands.add(Integer.toString(np)); // no. of containers commands.add("--rank"); commands.add(" " + Integer.toString(rank++)); // rank //temp sock port to share rank and ports commands.add("--wireUpPort"); commands.add(wireUpPort); if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } ctx.setCommands(commands); // Set local resource for containers ctx.setLocalResources(localResources); // Set environment for container Map<String, String> containerEnv = new HashMap<String, String>(); setupEnv(containerEnv); ctx.setEnvironment(containerEnv); // Time to start the container nmClient.startContainer(container, ctx); } while (completedContainers < np) { // argument to allocate() is the progress indicator AllocateResponse response = rmClient.allocate(completedContainers / np); for (ContainerStatus status : response.getCompletedContainersStatuses()) { if (debugYarn) { System.out.println("\n[MPJAppMaster]: Container Id - " + status.getContainerId()); System.out.println("[MPJAppMaster]: Container State - " + status.getState().toString()); System.out.println("[MPJAppMaster]: Container Diagnostics - " + status.getDiagnostics()); } ++completedContainers; } if (completedContainers != np) { Thread.sleep(100); } ; } // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); //shutDown AppMaster IO System.out.println("EXIT"); }