List of usage examples for org.apache.hadoop.yarn.api.records Priority setPriority
@Public @Stable public abstract void setPriority(int priority);
From source file:org.apache.hoya.yarn.appmaster.state.ContainerPriority.java
License:Apache License
public static Priority createPriority(int role, boolean locationSpecified) { Priority pri = Records.newRecord(Priority.class); pri.setPriority(ContainerPriority.buildPriority(role, locationSpecified)); return pri;//from www. j a va2 s .c o m }
From source file:org.apache.ignite.yarn.ApplicationMaster.java
License:Apache License
/** * Runs application master.//w w w.j a va2 s. c o m * * @throws Exception If failed. */ public void run() throws Exception { // Register with ResourceManager rmClient.registerApplicationMaster("", 0, ""); log.log(Level.INFO, "Application master registered."); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); try { // Check ignite cluster. while (!nmClient.isInState(Service.STATE.STOPPED)) { int runningCnt = containers.size(); if (runningCnt < props.instances() && checkAvailableResource()) { // Resource requirements for worker containers. Resource capability = Records.newRecord(Resource.class); capability.setMemory((int) props.totalMemoryPerNode()); capability.setVirtualCores((int) props.cpusPerNode()); for (int i = 0; i < props.instances() - runningCnt; ++i) { // Make container requests to ResourceManager AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerAsk); log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.", new Object[] { props.totalMemoryPerNode(), props.cpusPerNode() }); } } TimeUnit.MILLISECONDS.sleep(schedulerTimeout); } } catch (InterruptedException ignored) { // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.KILLED, "", ""); log.log(Level.WARNING, "Application master killed."); } catch (Exception e) { // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED, "", ""); log.log(Level.SEVERE, "Application master failed.", e); } }
From source file:org.apache.reef.runtime.yarn.driver.YarnResourceRequestHandler.java
License:Apache License
private synchronized Priority getPriority( final DriverRuntimeProtocol.ResourceRequestProto resourceRequestProto) { final Priority pri = Records.newRecord(Priority.class); pri.setPriority(resourceRequestProto.hasPriority() ? resourceRequestProto.getPriority() : 1); return pri;//from ww w . j a v a 2 s . c om }
From source file:org.apache.slider.core.launch.AppMasterLauncher.java
License:Apache License
/** * Complete the launch context (copy in env vars, etc). * @return the container to launch//w ww. j a v a2 s. com */ public ApplicationSubmissionContext completeAppMasterLaunch() throws IOException { //queue priority Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); submissionContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM // Queue for App master submissionContext.setQueue(queue); //container requirements submissionContext.setResource(resource); submissionContext.setLogAggregationContext(logAggregationContext); if (keepContainersOverRestarts) { log.debug("Requesting cluster stays running over AM failure"); submissionContext.setKeepContainersAcrossApplicationAttempts(true); } if (maxAppAttempts > 0) { log.debug("Setting max AM attempts to {}", maxAppAttempts); submissionContext.setMaxAppAttempts(maxAppAttempts); } if (secureCluster) { addSecurityTokens(); } else { propagateUsernameInInsecureCluster(); } completeContainerLaunch(); submissionContext.setAMContainerSpec(containerLaunchContext); return submissionContext; }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress) throws IOException, YarnException { QueryId queryId = queryInProgress.getQueryId(); ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId); LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" + queryId + ", appId=" + appId); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId);//w w w.ja va 2s .c o m // set the application name appContext.setApplicationName("Tajo"); Priority pri = Records.newRecord(Priority.class); pri.setPriority(5); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); ContainerLaunchContext commonContainerLaunchContext = YarnContainerProxy .createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true); // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m"); for (String eachToken : jvmOptions.split((" "))) { vargs.add(eachToken); } // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("qm"); vargs.add(queryId.toString()); // queryId vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":" + masterContext.getTajoMasterService().getBindAddress().getPort()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up QueryMasterRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); final Resource resource = Records.newRecord(Resource.class); // TODO - get default value from conf resource.setMemory(2000); resource.setVirtualCores(1); Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext( commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>(2)); appContext.setAMContainerSpec(masterContainerContext); LOG.info("Submitting QueryMaster to ResourceManager"); yarnClient.submitApplication(appContext); ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching QueryMaster with appAttemptId: " + attemptId); return attemptId; }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
License:Apache License
/** * Main getLaunchContext function for launch this application * * @return true if application completed successfully * @throws java.io.IOException/*from w w w .ja va 2s . c o m*/ * @throws org.apache.hadoop.yarn.exceptions.YarnException */ private void launch() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); displayClusterSummary(); // Get a new application id YarnClientApplication app = yarnClient.createApplication(); // validate resource capacity for launch an application amster validateResourceForAM(app); // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); // Set up the container launch context for the application master setupAMContainerLaunchContext(appContext); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); capability.setVirtualCores(amVCores); appContext.setResource(capability); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application // return monitorApplication(appId); }
From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM *///from w w w .j av a 2 s . c o m @Override public TajoContainerRequest getContainerRequest() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request Priority pri = Records.newRecord(Priority.class); pri.setPriority(appContext.getRequestPriority()); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(appContext.getWorkerMemory()); capability.setVirtualCores(appContext.getWorkerVCores()); TajoContainerRequest request = new TajoContainerRequest(capability, null, null, pri, this); LOG.info("Requested QueryMaster container ask: " + request.toString()); return request; }
From source file:org.chenchun.ApplicationMaster.java
License:Apache License
public static void main(String[] args) throws IOException, YarnException, InterruptedException { final String params = args[0]; final int containerNum = Integer.valueOf(args[1]); // Initialize clients to ResourceManager and NodeManagers Configuration conf = new YarnConfiguration(); AMRMClient<AMRMClient.ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf);/* w w w . j av a 2 s . com*/ rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager System.out.println("registerApplicationMaster 0"); rmClient.registerApplicationMaster("", 0, ""); System.out.println("registerApplicationMaster 1"); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(128); capability.setVirtualCores(1); // Make container requests to ResourceManager for (int i = 0; i < containerNum; ++i) { AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null, priority); System.out.println("Making res-req " + i); rmClient.addContainerRequest(containerAsk); } // Obtain allocated containers, launch and check for responses int responseId = 0; int completedContainers = 0; while (completedContainers < containerNum) { AllocateResponse response = rmClient.allocate(responseId++); System.out.println("Allocate response " + response.getAMCommand() + " " + "allocate " + response.getAllocatedContainers().size() + "contains"); for (Container container : response.getAllocatedContainers()) { // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setCommands( Collections.singletonList(params + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); System.out.println("Launching container " + container.getId() + " on " + container.getNodeId()); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { ++completedContainers; System.out.println("Completed container " + status.getContainerId()); } Thread.sleep(1000); } System.out.println("Unregister ApplicationMaster"); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }
From source file:org.deeplearning4j.iterativereduce.runtime.Utils.java
License:Apache License
public static ResourceRequest createResourceRequest(String host, int amount, int memory) { ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class); rsrcRequest.setHostName(host);//from w w w .j ava2s.co m Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); rsrcRequest.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); rsrcRequest.setCapability(capability); rsrcRequest.setNumContainers(amount); LOG.debug("Created a resource request" + ", host=" + rsrcRequest.getHostName() + ", memory=" + rsrcRequest.getCapability().getMemory() + ", amount=" + rsrcRequest.getNumContainers() + ", priority=" + rsrcRequest.getPriority().getPriority()); return rsrcRequest; }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java
License:Apache License
public void submitApplication(ApplicationId appId, String appName, Map<String, String> env, Map<String, LocalResource> localResources, List<String> commands, int memory) throws URISyntaxException, IOException { if (clientResourceManager == null) throw new IllegalStateException("Cannot submit an application without connecting to resource manager!"); ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class); appCtx.setApplicationId(appId);/*from ww w . j a v a 2s . c o m*/ appCtx.setApplicationName(appName); appCtx.setQueue("default"); appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); //System.out.println( "Based on my current user I am: " + UserGroupInformation.getCurrentUser().getShortUserName() ); Priority prio = Records.newRecord(Priority.class); prio.setPriority(0); appCtx.setPriority(prio); // Launch ctx ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class); containerCtx.setLocalResources(localResources); containerCtx.setCommands(commands); containerCtx.setEnvironment(env); containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); containerCtx.setResource(capability); appCtx.setAMContainerSpec(containerCtx); SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class); submitReq.setApplicationSubmissionContext(appCtx); LOG.info("Submitting application to ASM"); clientResourceManager.submitApplication(submitReq); // Don't return anything, ASM#submit returns an empty response }