List of usage examples for org.apache.hadoop.yarn.api.records Priority newInstance
@Public
@Stable
public static Priority newInstance(int p)
From source file:ApplicationMaster.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM *//*from w w w . j ava2 s .c o m*/ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * Returns an argument matcher which matches the expected worker container request for the * specified hosts./*from w w w. j a v a 2 s. co m*/ * * @param hosts the hosts in the container request * @return the argument matcher */ private ArgumentMatcher<ContainerRequest> getWorkerContainerMatcher(final List<String> hosts) { return new ArgumentMatcher<ContainerRequest>() { public boolean matches(Object arg) { Assert.assertTrue(arg instanceof ContainerRequest); ContainerRequest argContainer = (ContainerRequest) arg; // Wrap hosts with Sets to ignore ordering return argContainer.getCapability() .equals(Resource.newInstance(WORKER_MEM_MB + RAMDISK_MEM_MB, WORKER_CPU)) && Sets.newHashSet(argContainer.getNodes()).equals(Sets.newHashSet(hosts)) && argContainer.getRacks() == null && argContainer.getPriority().equals(Priority.newInstance(1)) && !argContainer.getRelaxLocality(); } }; }
From source file:alluxio.yarn.ApplicationMasterTest.java
License:Apache License
/** * Returns an argument matcher which matches the expected master container request. * * @return the argument matcher/*from w ww .j a v a2 s . co m*/ */ private ArgumentMatcher<ContainerRequest> getMasterContainerMatcher() { return new ArgumentMatcher<ContainerRequest>() { public boolean matches(Object arg) { boolean requireLocality = MASTER_ADDRESS.equals("localhost"); ContainerRequest expectedWorkerContainerRequest = new ContainerRequest( Resource.newInstance(MASTER_MEM_MB, MASTER_CPU), new String[] { MASTER_ADDRESS }, null, Priority.newInstance(0), requireLocality); return EqualsBuilder.reflectionEquals(arg, expectedWorkerContainerRequest); } }; }
From source file:alluxio.yarn.Client.java
License:Apache License
/** * Sets up the application submission context. *///from w w w . ja v a 2s. c o m private void setupApplicationSubmissionContext() { // set the application name mAppContext.setApplicationName(mAppName); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and vcores requirements Resource capability = Resource.newInstance(mAmMemoryInMB, mAmVCores); mAppContext.setResource(capability); // Set the queue to which this application is to be submitted in the RM mAppContext.setQueue(mAmQueue); // Set the AM container spec mAppContext.setAMContainerSpec(mAmContainer); // Set the priority for the application master mAppContext.setPriority(Priority.newInstance(mAmPriority)); }
From source file:alluxio.yarn.ContainerAllocator.java
License:Apache License
private void requestContainers(int numContainersToRequest) throws Exception { LOG.info("Requesting {} {} containers", numContainersToRequest, mContainerName); String[] hosts;/*from w w w . jav a 2s. com*/ boolean relaxLocality; // YARN requires that priority for relaxed-locality requests is different from strict-locality. Priority priority; if (mPreferredHost != null) { hosts = new String[] { mPreferredHost }; relaxLocality = mPreferredHost.equals("any"); priority = Priority.newInstance(100); } else { hosts = getPotentialWorkerHosts(); relaxLocality = true; priority = Priority.newInstance(101); } if (hosts.length * mMaxContainersPerHost < numContainersToRequest) { throw new RuntimeException(ExceptionMessage.YARN_NOT_ENOUGH_HOSTS.getMessage(numContainersToRequest, mContainerName, hosts.length)); } ContainerRequest containerRequest = new ContainerRequest(mResource, hosts, null /* any racks */, priority, relaxLocality); LOG.info("Making {} resource request(s) for Alluxio {}s with cpu {} memory {}MB on hosts {}", numContainersToRequest, mContainerName, mResource.getVirtualCores(), mResource.getMemory(), hosts); for (int i = 0; i < numContainersToRequest; i++) { mRMClient.addContainerRequest(containerRequest); } }
From source file:cn.edu.buaa.act.petuumOnYarn.ApplicationMaster.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM *//*from ww w . j a v a2 s .c om*/ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // set the priority for the request Priority pri = Priority.newInstance(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu // requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); String[] nodes = null; if (!avaliableNodeList.isEmpty()) { nodes = new String[1]; nodes[0] = (String) avaliableNodeList.get(0).getNodeId().getHost(); avaliableNodeList.remove(0); } //String[] racks = {"/default-rack"}; ContainerRequest request = new ContainerRequest(capability, nodes, null, pri, false); LOG.info("Requested container ask: " + request.toString() + ", nodes: " + request.getNodes()); return request; }
From source file:cn.edu.buaa.act.petuumOnYarn.Client.java
License:Apache License
/** * Main run function for the client//from w ww. ja v a2 s . c o m * * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); String[] s; s = conf.getStrings(YarnConfiguration.RM_ADDRESS); for (String ss : s) LOG.info("RM address: " + ss); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers() + ", nodeIdHost" + node.getNodeId().getHost()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of // the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); YarnUtil.copyAndAddToLocalResources(fs, appMasterJar, petuumHDFSPathPrefix, appMasterJarPath, localResources, null); scriptHDFSPath = YarnUtil.copyToHDFS(fs, scriptPath, petuumHDFSPathPrefix, launchPath, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { YarnUtil.copyAndAddToLocalResources(fs, log4jPropFile, petuumHDFSPathPrefix, log4jPath, localResources, null); } // Set the env variables to be setup in the env where the application // master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_nodes " + String.valueOf(numNodes)); vargs.add("--start_port " + String.valueOf(startPort)); vargs.add("--priority " + String.valueOf(workerPriority)); vargs.add("--script_hdfs_path " + scriptHDFSPath); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and // MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = // applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on // success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application currentTime = System.currentTimeMillis(); LOG.info("submit AM in " + (currentTime - startTime) + "ms"); return monitorApplication(appId); }
From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java
License:Apache License
/** * YARN only allows one resource size per priority, so map resource sizes * to priorities./*from w ww .j a va2s . c om*/ * Should be able to remove this when YARN-314 is fixed and choose purely on * locality. */ static Priority getRequestPriority(int mbs, int vcores, com.cloudera.llama.am.api.Resource.Locality locality) { // Lower values mean higher priority // More restrictive localities should get higher priority because they are // harder to satisfy // Higher values should get higher priority because they are harder to satisfy // Giving memory priority over CPU isn't ideal, but the alternative isn't any better int priority; switch (locality) { case MUST: priority = 1000000; break; case PREFERRED: priority = 2000000; break; default: priority = 3000000; break; } priority -= mbs; priority -= vcores; return Priority.newInstance(priority); }
From source file:com.datatorrent.stram.ResourceRequestHandler.java
License:Apache License
/** * Setup the request(s) that will be sent to the RM for the container ask. *//* w w w . j a v a 2s . c om*/ public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first) { int priority = csr.container.getResourceRequestPriority(); // check for node locality constraint String[] nodes = null; String[] racks = null; String host = getHost(csr, first); Resource capability = Records.newRecord(Resource.class); capability.setMemory(csr.container.getRequiredMemoryMB()); capability.setVirtualCores(csr.container.getRequiredVCores()); if (host != null) { nodes = new String[] { host }; // in order to request a host, we don't have to set the rack if the locality is false /* * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; } */ return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false); } // For now, only memory is supported so we set memory requirements return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority)); }
From source file:com.datatorrent.stram.StreamingAppMasterService.java
License:Apache License
/** * Check for containers that were allocated in a previous attempt. * If the containers are still alive, wait for them to check in via heartbeat. *//*from w w w. jav a2 s . c o m*/ private void checkContainerStatus() { Collection<StreamingContainerAgent> containers = this.dnmgr.getContainerAgents(); for (StreamingContainerAgent ca : containers) { ContainerId containerId = ConverterUtils.toContainerId(ca.container.getExternalId()); NodeId nodeId = ConverterUtils.toNodeId(ca.container.host); // put container back into the allocated list org.apache.hadoop.yarn.api.records.Token containerToken = null; Resource resource = Resource.newInstance(ca.container.getAllocatedMemoryMB(), ca.container.getAllocatedVCores()); Priority priority = Priority.newInstance(ca.container.getResourceRequestPriority()); Container yarnContainer = Container.newInstance(containerId, nodeId, ca.container.nodeHttpAddress, resource, priority, containerToken); this.allocatedContainers.put(containerId.toString(), new AllocatedContainer(yarnContainer)); // check the status nmClient.getContainerStatusAsync(containerId, nodeId); } }