List of usage examples for org.apache.hadoop.yarn.client.api.async AMRMClientAsync start
@Override public void start()
From source file:com.gpiskas.yarn.AppMaster.java
License:Open Source License
public void run() throws Exception { conf = new YarnConfiguration(); // Create NM Client nmClient = NMClient.createNMClient(); nmClient.init(conf);// w w w. j a v a 2 s . com nmClient.start(); // Create AM - RM Client AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this); rmClient.init(conf); rmClient.start(); // Register with RM rmClient.registerApplicationMaster("", 0, ""); System.out.println("AppMaster: Registered"); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(128); capability.setVirtualCores(1); // Reqiest Containers from RM System.out.println("AppMaster: Requesting " + containerCount + " Containers"); for (int i = 0; i < containerCount; ++i) { rmClient.addContainerRequest(new ContainerRequest(capability, null, null, priority)); } while (!containersFinished()) { Thread.sleep(100); } System.out.println("AppMaster: Unregistered"); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }
From source file:hws.core.JobMaster.java
License:Apache License
public void runMainLoop() throws Exception { AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(100, this); rmClient.init(getConfiguration());//from w w w . ja va 2 s .c om rmClient.start(); // Register with ResourceManager Logger.info("[AM] registerApplicationMaster 0"); rmClient.registerApplicationMaster("", 0, ""); Logger.info("[AM] registerApplicationMaster 1"); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(128); capability.setVirtualCores(1); final CountDownLatch doneLatch = new CountDownLatch(this.modulePipeline.size()); // Make container requests to ResourceManager for (ModuleInfo moduleInfo : this.modulePipeline) { //create containers for each instance of each module zk.createPersistent("/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name(), ""); zk.createPersistent( "/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name() + "/finish", ""); zk.createPersistent( "/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name() + "/halted", ""); zk.subscribeChildChanges( "/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name() + "/finish", createFinishListener(moduleInfo.filterInfo().name(), moduleInfo.numFilterInstances(), doneLatch)); for (int i = 0; i < moduleInfo.numFilterInstances(); i++) { this.numContainersToWaitFor++; ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); Logger.info("[AM] Making res-req for " + moduleInfo.filterInfo().name() + " " + i); rmClient.addContainerRequest(containerAsk); } } //TODO: process for starting the whole application //create containers // -> create instances // -> start output channels and filters // -> start input channels in reversed topological order (considering that there is no cycle) // * if there is cycle, then inicially start in any order //TODO "send" the start signal via ZooKeeper Logger.info("[AM] waiting for containers to finish"); try { doneLatch.await(); //await the input threads to finish } catch (InterruptedException e) { Logger.fatal(e.toString()); //e.printStackTrace(); } /*while(!doneWithContainers()) { Thread.sleep(50); }*/ zk.createPersistent("/hadoop-watershed/" + appIdStr + "/done", ""); Logger.info("[AM] unregisterApplicationMaster 0"); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); Logger.info("[AM] unregisterApplicationMaster 1"); }
From source file:org.apache.reef.runtime.yarn.driver.unmanaged.UnmanagedAmTest.java
License:Apache License
@Test public void testAmShutdown() throws IOException, YarnException { Assume.assumeTrue("This test requires a YARN Resource Manager to connect to", Boolean.parseBoolean(System.getenv("REEF_TEST_YARN"))); final YarnConfiguration yarnConfig = new YarnConfiguration(); // Start YARN client and register the application final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfig);/*from w w w. j a v a 2 s .c om*/ yarnClient.start(); final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class); containerContext.setCommands(Collections.<String>emptyList()); containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap()); containerContext.setEnvironment(Collections.<String, String>emptyMap()); containerContext.setTokens(getTokens()); final ApplicationSubmissionContext appContext = yarnClient.createApplication() .getApplicationSubmissionContext(); appContext.setApplicationName("REEF_Unmanaged_AM_Test"); appContext.setAMContainerSpec(containerContext); appContext.setUnmanagedAM(true); appContext.setQueue("default"); final ApplicationId applicationId = appContext.getApplicationId(); LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId); yarnClient.submitApplication(appContext); LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId); addToken(yarnClient.getAMRMToken(applicationId)); // Start the AM final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this); rmClient.init(yarnConfig); rmClient.start(); final NMClientAsync nmClient = new NMClientAsyncImpl(this); nmClient.init(yarnConfig); nmClient.start(); final RegisterApplicationMasterResponse registration = rmClient .registerApplicationMaster(NetUtils.getHostname(), -1, null); LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null); LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState()); // Shutdown the AM rmClient.stop(); nmClient.stop(); // Get the final application report final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); final YarnApplicationState appState = appReport.getYarnApplicationState(); final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus(); LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus }); Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState); Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus); // Shutdown YARN client yarnClient.stop(); }