List of usage examples for org.apache.hadoop.yarn.client.api YarnClient createYarnClient
@Public public static YarnClient createYarnClient()
From source file:org.testifyproject.resource.yarn.MiniYarnResource.java
License:Apache License
@Override public LocalResourceInstance<MiniYARNCluster, YarnClient> start(TestContext testContext, LocalResource localResource, YarnConfiguration config) throws Exception { String logDirectory = config.get(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); fileSystemUtil.recreateDirectory(logDirectory); server = new MiniYARNCluster(testContext.getName(), 1, 1, 1, 1, true); server.init(config);/* www .j a v a 2 s . com*/ server.start(); client = YarnClient.createYarnClient(); client.init(server.getConfig()); client.start(); return LocalResourceInstanceBuilder.builder().resource(server).client(client).build("yarn", localResource); }
From source file:org.trustedanalytics.servicebroker.gearpump.yarn.YarnClientFactory.java
License:Apache License
public YarnClient getYarnClient() throws IOException, LoginException { kerberosService.login();/*from w ww .j a v a 2 s . c o m*/ String ticketCachePath = yarnConfiguration.get(CommonConfigurationKeys.KERBEROS_TICKET_CACHE_PATH); UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, kerberosService.getKerberosProperties().getUser()); YarnClient yarnClient = new DelegatingYarnClient(YarnClient.createYarnClient(), new UgiWrapper(ugi)); yarnClient.init(yarnConfiguration); yarnClient.start(); return yarnClient; }
From source file:org.trustedanalytics.servicebroker.h2oprovisioner.cdhclients.DeprovisionerYarnClientProvider.java
License:Apache License
public DeprovisionerYarnClient getClient(String user, Configuration hadoopConf) throws IOException { String ticketCachePath = hadoopConf.get("hadoop.security.kerberos.ticket.cache.path"); UserGroupInformation ugi = UserGroupInformation.getBestUGI(ticketCachePath, user); UserGroupInformation.setLoginUser(ugi); YarnClient client = new DelegatingYarnClient(YarnClient.createYarnClient(), new UgiWrapper(ugi)); client.init(hadoopConf);/*from ww w. j a v a2s .c o m*/ return new DeprovisionerYarnClient(client); }
From source file:oz.hadoop.yarn.api.core.ApplicationMasterLauncherImpl.java
License:Apache License
/** * /*from w ww . ja va2 s. com*/ * @param applicationSpecification */ ApplicationMasterLauncherImpl(Map<String, Object> applicationSpecification) { super(applicationSpecification); this.yarnClient = YarnClient.createYarnClient(); this.initializeJarExclusionPatterns(); }
From source file:proxyyarn.ProxyYarn.java
License:Apache License
public boolean run() throws Exception { Configuration conf = new YarnConfiguration(new Configuration()); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);//from ww w .j a va2s . co m yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); log.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); log.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { log.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo("default"); log.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { log.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } FileSystem fs = FileSystem.get(conf); if (!fs.getClass().equals(DistributedFileSystem.class)) { log.error("Expected DistributedFileSystem, but was {}", fs.getClass().getSimpleName()); System.exit(1); } // ApplicationClientProtocol applicationsManager; // InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); // log.info("Connecting to ResourceManager at {}", rmAddress); // Configuration appManagerServerConf = new Configuration(conf); // YarnRPC rpc = YarnRPC.create(appManagerServerConf); // ApplicationClientProtocol applicationManager = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, appManagerServerConf); String appName = "AccumuloProxyYarn"; YarnClientApplication app = yarnClient.createApplication(); // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class); // GetNewApplicationResponse response = applicationManager.getNewApplication(request); // log.info("Got new ApplicationId=" + response.getApplicationId()); // ApplicationId appId = response.getApplicationId(); // Create a new ApplicationSubmissionContext // ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the ApplicationId // appContext.setApplicationId(appId); // set the application name // appContext.setApplicationName(appName); // Create a new container launch context for the AM's container ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // Define the local resources required Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // Lets assume the jar we need for our ApplicationMaster is available in // HDFS at a certain known path to us and we want to make it available to // the ApplicationMaster in the launched container Path localJarPath = new Path( "file:///Users/jelser/projects/accumulo-proxy-yarn/target/accumulo-proxy-yarn-0.0.1-SNAPSHOT.jar"); Path jarPath = new Path("hdfs:///accumulo-proxy-yarn-0.0.1-SNAPSHOT.jar"); fs.copyFromLocalFile(false, true, localJarPath, jarPath); FileStatus jarStatus = fs.getFileStatus(jarPath); LocalResource amJarRsrc = Records.newRecord(LocalResource.class); // Set the type of resource - file or archive // archives are untarred at the destination by the framework amJarRsrc.setType(LocalResourceType.FILE); // Set visibility of the resource // Setting to most private option i.e. this file will only // be visible to this instance of the running application amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION); // Set the location of resource to be copied over into the // working directory amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application amJarRsrc.setTimestamp(jarStatus.getModificationTime()); amJarRsrc.setSize(jarStatus.getLen()); // The framework will create a symlink called AppMaster.jar in the // working directory that will be linked back to the actual file. // The ApplicationMaster, if needs to reference the jar file, would // need to use the symlink filename. localResources.put("AppMaster.jar", amJarRsrc); // Set the local resources into the launch context amContainer.setLocalResources(localResources); // Set up the environment needed for the launch context Map<String, String> env = new HashMap<String, String>(); // For example, we could setup the classpath needed. // Assuming our classes or jars are available as local resources in the // working directory from which the command will be run, we need to append // "." to the path. // By default, all the hadoop specific classpaths will already be available // in $CLASSPATH, so we should be careful not to overwrite it. String classPathEnv = "$CLASSPATH:./*:/Users/jelser/projects/accumulo-proxy-yarn/target/lib/*"; env.put("CLASSPATH", classPathEnv); amContainer.setEnvironment(env); // Construct the command to be executed on the launched container String command = "${JAVA_HOME}" + "/bin/java" + " proxyyarn.ProxyYarnAppMaster 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; List<String> commands = new ArrayList<String>(); commands.add(command); // add additional commands if needed // Set the command array into the container spec amContainer.setCommands(commands); // Define the resource requirements for the container // For now, YARN only supports memory so we set the memory // requirements. // If the process takes more than its allocated memory, it will // be killed by the framework. // Memory being requested for should be less than max capability // of the cluster and all asks should be a multiple of the min capability. Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); appContext.setResource(capability); // Create the request to send to the ApplicationsManager // SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); // appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the ApplicationsManager // Ignore the response as either a valid response object is returned on // success or an exception thrown to denote the failure // applicationManager.submitApplication(appRequest); // Set the container launch content into the ApplicationSubmissionContext appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure log.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); return monitorApplication(yarnClient, appId); /* Thread.sleep(200); boolean running = false; while(true) { GetApplicationReportRequest reportRequest = Records.newRecord(GetApplicationReportRequest.class); reportRequest.setApplicationId(appId); GetApplicationReportResponse reportResponse = applicationManager.getApplicationReport(reportRequest); ApplicationReport report = reportResponse.getApplicationReport(); log.info(report.toString()); YarnApplicationState state = report.getYarnApplicationState(); switch (state) { case NEW: case NEW_SAVING: case SUBMITTED: case ACCEPTED: log.info("State: {}", state); break; case RUNNING: log.info("Running application"); running = true; break; case FINISHED: case FAILED: case KILLED: log.info("State: {}", state); return; default: log.info("Unknown state: {}", state); return; } if (!running) { Thread.sleep(1000); } }*/ }
From source file:runtime.starter.MPJYarnClient.java
License:Open Source License
public void run() throws Exception { Map<String, String> map = System.getenv(); try {// www . ja v a 2s . co m mpjHomeDir = map.get("MPJ_HOME"); if (mpjHomeDir == null) { throw new Exception("[MPJRun.java]:MPJ_HOME environment found.."); } } catch (Exception exc) { System.out.println("[MPJRun.java]:" + exc.getMessage()); exc.printStackTrace(); return; } // Copy the application master jar to HDFS // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); /* Path dataset = new Path(fs.getHomeDirectory(),"/dataset"); FileStatus datasetFile = fs.getFileStatus(dataset); BlockLocation myBlocks [] = fs.getFileBlockLocations(datasetFile,0,datasetFile.getLen()); for(BlockLocation b : myBlocks){ System.out.println("\n--------------------"); System.out.println("Length "+b.getLength()); for(String host : b.getHosts()){ System.out.println("host "+host); } } */ Path source = new Path(mpjHomeDir + "/lib/mpj-app-master.jar"); String pathSuffix = hdfsFolder + "mpj-app-master.jar"; Path dest = new Path(fs.getHomeDirectory(), pathSuffix); if (debugYarn) { logger.info("Uploading mpj-app-master.jar to: " + dest.toString()); } fs.copyFromLocalFile(false, true, source, dest); FileStatus destStatus = fs.getFileStatus(dest); Path wrapperSource = new Path(mpjHomeDir + "/lib/mpj-yarn-wrapper.jar"); String wrapperSuffix = hdfsFolder + "mpj-yarn-wrapper.jar"; Path wrapperDest = new Path(fs.getHomeDirectory(), wrapperSuffix); if (debugYarn) { logger.info("Uploading mpj-yarn-wrapper.jar to: " + wrapperDest.toString()); } fs.copyFromLocalFile(false, true, wrapperSource, wrapperDest); Path userJar = new Path(jarPath); String userJarSuffix = hdfsFolder + "user-code.jar"; Path userJarDest = new Path(fs.getHomeDirectory(), userJarSuffix); if (debugYarn) { logger.info("Uploading user-code.jar to: " + userJarDest.toString()); } fs.copyFromLocalFile(false, true, userJar, userJarDest); YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); if (debugYarn) { YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); logger.info("\nNodes Information"); logger.info("Number of NM: " + metrics.getNumNodeManagers() + "\n"); List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport n : nodeReports) { logger.info("NodeId: " + n.getNodeId()); logger.info("RackName: " + n.getRackName()); logger.info("Total Memory: " + n.getCapability().getMemory()); logger.info("Used Memory: " + n.getUsed().getMemory()); logger.info("Total vCores: " + n.getCapability().getVirtualCores()); logger.info("Used vCores: " + n.getUsed().getVirtualCores() + "\n"); } } logger.info("Creating server socket at HOST " + serverName + " PORT " + serverPort + " \nWaiting for " + np + " processes to connect..."); // Creating a server socket for incoming connections try { servSock = new ServerSocket(serverPort); infoSock = new ServerSocket(); TEMP_PORT = findPort(infoSock); } catch (Exception e) { e.printStackTrace(); } // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { logger.info("Max memory capability resources in cluster: " + maxMem); } if (amMem > maxMem) { amMem = maxMem; logger.info("AM memory specified above threshold of cluster " + "Using maximum memory for AM container: " + amMem); } int maxVcores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { logger.info("Max vCores capability resources in cluster: " + maxVcores); } if (amCores > maxVcores) { amCores = maxVcores; logger.info("AM virtual cores specified above threshold of cluster " + "Using maximum virtual cores for AM container: " + amCores); } // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add("$JAVA_HOME/bin/java"); commands.add("-Xmx" + amMem + "m"); commands.add("runtime.starter.MPJAppMaster"); commands.add("--np"); commands.add(String.valueOf(np)); commands.add("--serverName"); commands.add(serverName); //server name commands.add("--ioServerPort"); commands.add(Integer.toString(serverPort)); //server port commands.add("--deviceName"); commands.add(deviceName); //device name commands.add("--className"); commands.add(className); //class name commands.add("--wdir"); commands.add(workingDirectory); //wdir commands.add("--psl"); commands.add(Integer.toString(psl)); //protocol switch limit commands.add("--wireUpPort"); commands.add(String.valueOf(TEMP_PORT)); //for sharing ports & rank commands.add("--wrapperPath"); commands.add(wrapperDest.toString());//MPJYarnWrapper.jar HDFS path commands.add("--userJarPath"); commands.add(userJarDest.toString());//User Jar File HDFS path commands.add("--mpjContainerPriority"); commands.add(mpjContainerPriority);// priority for mpj containers commands.add("--containerMem"); commands.add(containerMem); commands.add("--containerCores"); commands.add(containerCores); if (debugYarn) { commands.add("--debugYarn"); } if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } amContainer.setCommands(commands); //set commands // Setup local Resource for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(dest)); appMasterJar.setSize(destStatus.getLen()); appMasterJar.setTimestamp(destStatus.getModificationTime()); appMasterJar.setType(LocalResourceType.ARCHIVE); appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION); amContainer.setLocalResources(Collections.singletonMap("mpj-app-master.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster // Setting up the environment Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMem); capability.setVirtualCores(amCores); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(yarnQueue); // queue Priority priority = Priority.newInstance(amPriority); appContext.setPriority(priority); ApplicationId appId = appContext.getApplicationId(); //Adding ShutDown Hook Runtime.getRuntime().addShutdownHook(new KillYarnApp(appId, yarnClient)); // Submit application System.out.println("Submitting Application: " + appContext.getApplicationName() + "\n"); try { isRunning = true; yarnClient.submitApplication(appContext); } catch (Exception exp) { System.err.println("Error Submitting Application"); exp.printStackTrace(); } // np = number of processes , + 1 for Application Master container IOMessagesThread[] ioThreads = new IOMessagesThread[np + 1]; peers = new String[np]; socketList = new Vector<Socket>(); int wport = 0; int rport = 0; int rank = 0; // np + 1 IOThreads for (int i = 0; i < (np + 1); i++) { try { sock = servSock.accept(); //start IO thread to read STDOUT and STDERR from wrappers IOMessagesThread io = new IOMessagesThread(sock); ioThreads[i] = io; ioThreads[i].start(); } catch (Exception e) { System.err.println("Error accepting connection from peer socket.."); e.printStackTrace(); } } // Loop to read port numbers from Wrapper.java processes // and to create WRAPPER_INFO (containing all IPs and ports) String WRAPPER_INFO = "#Peer Information"; for (int i = np; i > 0; i--) { try { sock = infoSock.accept(); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); DataInputStream in = new DataInputStream(sock.getInputStream()); if (in.readUTF().startsWith("Sending Info")) { wport = in.readInt(); rport = in.readInt(); rank = in.readInt(); peers[rank] = ";" + sock.getInetAddress().getHostAddress() + "@" + rport + "@" + wport + "@" + rank; socketList.add(sock); } } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error accepting" + " connection from peer socket!"); e.printStackTrace(); } } for (int i = 0; i < np; i++) { WRAPPER_INFO += peers[i]; } // Loop to broadcast WRAPPER_INFO to all Wrappers for (int i = np; i > 0; i--) { try { sock = socketList.get(np - i); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); out.writeUTF(WRAPPER_INFO); out.flush(); sock.close(); } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error closing" + " connection from peer socket.."); e.printStackTrace(); } } try { infoSock.close(); } catch (IOException exp) { exp.printStackTrace(); } // wait for all IO Threads to complete for (int i = 0; i < (np + 1); i++) { ioThreads[i].join(); } isRunning = true; System.out.println("\nApplication Statistics!"); while (true) { appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); fStatus = appReport.getFinalApplicationStatus(); if (appState == YarnApplicationState.FINISHED) { isRunning = false; if (fStatus == FinalApplicationStatus.SUCCEEDED) { System.out.println("State: " + fStatus); } else { System.out.println("State: " + fStatus); } break; } else if (appState == YarnApplicationState.KILLED) { isRunning = false; System.out.println("State: " + appState); break; } else if (appState == YarnApplicationState.FAILED) { isRunning = false; System.out.println("State: " + appState); break; } Thread.sleep(100); } try { if (debugYarn) { logger.info("Cleaning the files from hdfs: "); logger.info("1) " + dest.toString()); logger.info("2) " + wrapperDest.toString()); logger.info("3) " + userJarDest.toString()); } fs.delete(dest); fs.delete(wrapperDest); fs.delete(userJarDest); } catch (IOException exp) { exp.printStackTrace(); } System.out.println("Application ID: " + appId + "\n" + "Application User: " + appReport.getUser() + "\n" + "RM Queue: " + appReport.getQueue() + "\n" + "Start Time: " + appReport.getStartTime() + "\n" + "Finish Time: " + appReport.getFinishTime()); }
From source file:tachyon.yarn.ApplicationMasterTest.java
License:Apache License
private void setupApplicationMaster(Map<String, String> properties) throws Exception { TachyonConf conf = new TachyonConf(); for (Entry<String, String> entry : properties.entrySet()) { conf.set(entry.getKey(), entry.getValue()); }//from w ww . ja v a 2s. com mMaster = new ApplicationMaster(NUM_WORKERS, MASTER_ADDRESS, RESOURCE_ADDRESS, conf); mPrivateAccess = new ApplicationMasterPrivateAccess(mMaster); // Mock Node Manager client PowerMockito.mockStatic(NMClient.class); mNMClient = Mockito.mock(NMClient.class); Mockito.when(NMClient.createNMClient()).thenReturn(mNMClient); // Mock Application Master Resource Manager client PowerMockito.mockStatic(AMRMClientAsync.class); @SuppressWarnings("unchecked") AMRMClientAsync<ContainerRequest> amrm = (AMRMClientAsync<ContainerRequest>) Mockito .mock(AMRMClientAsync.class); mRMClient = amrm; Mockito.when(AMRMClientAsync.createAMRMClientAsync(100, mMaster)).thenReturn(mRMClient); // Mock Yarn client PowerMockito.mockStatic(YarnClient.class); mYarnClient = (YarnClient) Mockito.mock(YarnClient.class); Mockito.when(YarnClient.createYarnClient()).thenReturn(mYarnClient); // Partially mock Utils to avoid hdfs IO PowerMockito.mockStatic(YarnUtils.class); Mockito.when(YarnUtils.createLocalResourceOfFile(Mockito.<YarnConfiguration>any(), Mockito.anyString())) .thenReturn(Records.newRecord(LocalResource.class)); Mockito.when(YarnUtils.buildCommand(YarnContainerType.TACHYON_MASTER)).thenReturn(EXPECTED_MASTER_COMMAND); Mockito.when(YarnUtils.buildCommand(YarnContainerType.TACHYON_WORKER)).thenReturn(EXPECTED_WORKER_COMMAND); mMaster.start(); }
From source file:tachyon.yarn.Client.java
License:Apache License
/** * Submits an application to the ResourceManager to run ApplicationMaster. * * The stable Yarn API provides a convenience method (YarnClient#createApplication) for creating * applications and setting up the application submission context. This was not available in the * alpha API./* ww w .j a v a 2 s .c o m*/ */ private void submitApplication() throws YarnException, IOException { // TODO(binfan): setup credential // Initialize a YarnClient mYarnClient = YarnClient.createYarnClient(); mYarnClient.init(mYarnConf); mYarnClient.start(); // Create an application, get and check the information about the cluster YarnClientApplication app = mYarnClient.createApplication(); // Get a response of this application, containing information of the cluster GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // Check if the cluster has enough resource to launch the ApplicationMaster checkClusterResource(appResponse); // Set up the container launch context for the application master mAmContainer = Records.newRecord(ContainerLaunchContext.class); setupContainerLaunchContext(); // Finally, set-up ApplicationSubmissionContext for the application mAppContext = app.getApplicationSubmissionContext(); setupApplicationSubmissionContext(); // Submit the application to the applications manager. // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure mAppId = mAppContext.getApplicationId(); System.out.println("Submitting application of id " + mAppId + " to ResourceManager"); mYarnClient.submitApplication(mAppContext); }
From source file:timo.yarn_app_call_java_app.Client.java
License:Apache License
Client(String appMasterMainClass, Configuration conf) {
this.conf = conf;
this.appMasterMainClass = appMasterMainClass;
yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);/*from www . j ava 2s . c om*/
opts = new Options();
opts.addOption("priority", true, "Application Priority. Default 0");
opts.addOption("queue", true, "RM Queue in which this application is to be submitted");
opts.addOption("timeout", true, "Application timeout in milliseconds");
opts.addOption("master_memory", true,
"Amount of memory in MB to be requested to run the application master");
opts.addOption("master_vcores", true,
"Amount of virtual cores to be requested to run the application master");
opts.addOption("jar", true, "Jar file containing the application master");
opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the command");
opts.addOption("container_vcores", true, "Amount of virtual cores to be requested to run the command");
opts.addOption("num_containers", true, "No. of containers on which the command needs to be executed");
opts.addOption("log_properties", true, "log4j.properties file");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application attempts."
+ " If the flag is true, running containers will not be killed when"
+ " application attempt fails and these containers will be retrieved by"
+ " the new application attempt ");
opts.addOption("attempt_failures_validity_interval", true,
"when attempt_failures_validity_interval in milliseconds is set to > 0,"
+ "the failure number will not take failures which happen out of "
+ "the validityInterval into failure count. "
+ "If failure count reaches to maxAppAttempts, " + "the application will be failed.");
opts.addOption("debug", false, "Dump out debug information");
opts.addOption("domain", true, "ID of the timeline domain where the " + "timeline entities will be put");
opts.addOption("view_acls", true,
"Users and groups that allowed to " + "view the timeline entities in the given domain");
opts.addOption("modify_acls", true,
"Users and groups that allowed to " + "modify the timeline entities in the given domain");
opts.addOption("create", false,
"Flag to indicate whether to create the " + "domain specified with -domain.");
opts.addOption("help", false, "Print usage");
opts.addOption("node_label_expression", true,
"Node label expression to determine the nodes" + " where all the containers of this application"
+ " will be allocated, \"\" means containers"
+ " can be allocated anywhere, if you don't specify the option,"
+ " default node_label_expression of queue will be used.");
}
From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java
License:Open Source License
/** Kills the specified job. * @param jobId id of the job to be killed * @return 0 for success, -1 for no such job, -2 for job could not be killed * @throws Exception//from ww w . j a v a2 s . co m */ @Override public int killJob(final int jobId, boolean purge) throws Exception { UserGroupInformation caller = Server.getRemoteUser(); LOG.info(caller + " asked to kill job " + jobId); if (!jobArray.containsKey(jobId)) return -1; final JobInformation ji = jobArray.get(jobId); checkOwnerOrRoot(ji); UserGroupInformation proxyUser = ji.proxyUser; Integer status; PrivilegedExceptionAction<Integer> doKill = new PrivilegedExceptionAction<Integer>() { public Integer run() throws Exception { final long kill_deadline = System.currentTimeMillis() + pConf.getLong(PConfiguration.KEY_CONTROLLER_KILL_TIMEOUT, 5000); YarnClientService kittenClient = ji.kitten; YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yConf); yarnClient.start(); yarnClient.killApplication(kittenClient.getApplicationId()); while (!kittenClient.isApplicationFinished()) { Thread.sleep(100); if (System.currentTimeMillis() > kill_deadline) return -2; } return 0; } }; //perform the actual kill, as the user if (UserGroupInformation.isSecurityEnabled()) status = proxyUser.doAs(doKill); else status = doKill.run(); runningJobs.dec(); killedJobs.inc(); //purge, aka qdel -p. //conditional on superuser if (purge) { jobArray.remove(jobId); status = 0; } return status; }