List of usage examples for org.apache.hadoop.yarn.api.records Resource setVirtualCores
@Public @Evolving public abstract void setVirtualCores(int vCores);
From source file:org.apache.hoya.providers.hoyaam.HoyaAMClientProvider.java
License:Apache License
/** * Update the AM resource with any local needs * @param capability capability to update */// w ww .j a va 2 s .com public void prepareAMResourceRequirements(MapOperations hoyaAM, Resource capability) { capability.setMemory(hoyaAM.getOptionInt(ResourceKeys.YARN_MEMORY, capability.getMemory())); capability.setVirtualCores(hoyaAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); }
From source file:org.apache.hoya.yarn.appmaster.state.AppState.java
License:Apache License
/** * Build up the resource requirements for this role from the * cluster specification, including substituing max allowed values * if the specification asked for it.//from ww w . ja va2 s . co m * @param role role * @param capability capability to set up */ public void buildResourceRequirements(RoleStatus role, Resource capability) { // Set up resource requirements from role values String name = role.getName(); ConfTreeOperations resources = getResourcesSnapshot(); int cores = getResourceRequirement(resources, name, YARN_CORES, DEF_YARN_CORES, containerMaxCores); capability.setVirtualCores(cores); int ram = getResourceRequirement(resources, name, YARN_MEMORY, DEF_YARN_MEMORY, containerMaxMemory); capability.setMemory(ram); }
From source file:org.apache.ignite.yarn.ApplicationMaster.java
License:Apache License
/** * Runs application master./* www. j a v a 2 s .c o m*/ * * @throws Exception If failed. */ public void run() throws Exception { // Register with ResourceManager rmClient.registerApplicationMaster("", 0, ""); log.log(Level.INFO, "Application master registered."); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); try { // Check ignite cluster. while (!nmClient.isInState(Service.STATE.STOPPED)) { int runningCnt = containers.size(); if (runningCnt < props.instances() && checkAvailableResource()) { // Resource requirements for worker containers. Resource capability = Records.newRecord(Resource.class); capability.setMemory((int) props.totalMemoryPerNode()); capability.setVirtualCores((int) props.cpusPerNode()); for (int i = 0; i < props.instances() - runningCnt; ++i) { // Make container requests to ResourceManager AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerAsk); log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.", new Object[] { props.totalMemoryPerNode(), props.cpusPerNode() }); } } TimeUnit.MILLISECONDS.sleep(schedulerTimeout); } } catch (InterruptedException ignored) { // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.KILLED, "", ""); log.log(Level.WARNING, "Application master killed."); } catch (Exception e) { // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED, "", ""); log.log(Level.SEVERE, "Application master failed.", e); } }
From source file:org.apache.ignite.yarn.IgniteYarnClient.java
License:Apache License
/** * Main methods has one mandatory parameter and one optional parameter. * * @param args Path to jar mandatory parameter and property file is optional. *//*from ww w. j a va 2 s .c o m*/ public static void main(String[] args) throws Exception { checkArguments(args); // Set path to app master jar. String pathAppMasterJar = args[0]; ClusterProperties props = ClusterProperties.from(args.length == 2 ? args[1] : null); YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); FileSystem fs = FileSystem.get(conf); Path ignite; // Load ignite and jar if (props.ignitePath() == null) ignite = getIgnite(props, fs); else ignite = new Path(props.ignitePath()); // Upload the jar file to HDFS. Path appJar = IgniteYarnUtils.copyLocalToHdfs(fs, pathAppMasterJar, props.igniteWorkDir() + File.separator + IgniteYarnUtils.JAR_NAME); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands(Collections .singletonList(Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName() + IgniteYarnUtils.SPACE + ignite.toUri() + IgniteYarnUtils.YARN_LOG_OUT)); // Setup jar for ApplicationMaster LocalResource appMasterJar = IgniteYarnUtils.setupFile(appJar, fs, LocalResourceType.FILE); amContainer.setLocalResources(Collections.singletonMap(IgniteYarnUtils.JAR_NAME, appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = props.toEnvs(); setupAppMasterEnv(appMasterEnv, conf); amContainer.setEnvironment(appMasterEnv); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials creds = new Credentials(); String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokRenewer == null || tokRenewer.length() == 0) throw new IOException("Master Kerberos principal for the RM is not set."); log.info("Found RM principal: " + tokRenewer); final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds); if (tokens != null) log.info("File system delegation tokens: " + Arrays.toString(tokens)); amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds)); } // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(512); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("ignition"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); yarnClient.submitApplication(appContext); log.log(Level.INFO, "Submitted application. Application id: {0}", appId); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING || appState == YarnApplicationState.SUBMITTED || appState == YarnApplicationState.ACCEPTED) { TimeUnit.SECONDS.sleep(1L); appReport = yarnClient.getApplicationReport(appId); if (appState != YarnApplicationState.ACCEPTED && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) log.log(Level.INFO, "Application {0} is ACCEPTED.", appId); appState = appReport.getYarnApplicationState(); } log.log(Level.INFO, "Application {0} is {1}.", new Object[] { appId, appState }); }
From source file:org.apache.impala.yarn.server.utils.BuilderUtils.java
License:Apache License
public static Resource newResource(int memory, int vCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); resource.setMemory(memory);// w w w .j av a 2 s .co m resource.setVirtualCores(vCores); return resource; }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@Override public void beforeRMNodeEventHandled(RMNodeEvent event, RMContext context) { switch (event.getType()) { case STARTED: // Since the RMNode was just started, it should not have a non-zero capacity RMNode rmNode = context.getRMNodes().get(event.getNodeId()); if (isNonZeroCapacityNode(rmNode)) { Resource totalCapability = rmNode.getTotalCapability(); logger.warn(// w w w. ja va 2 s . c o m "FineGrainedScaling feature got invoked for a NM with non-zero capacity. Host: {}, Mem: {}, CPU: {}. Setting the " + "NM's capacity to (0G,0CPU)", rmNode.getHostName(), totalCapability.getMemory(), totalCapability.getVirtualCores()); totalCapability.setMemory(0); totalCapability.setVirtualCores(0); } break; case STATUS_UPDATE: handleStatusUpdate(event, context); break; default: break; } }
From source file:org.apache.reef.runtime.yarn.driver.YarnResourceRequestHandler.java
License:Apache License
private synchronized Resource getResource( final DriverRuntimeProtocol.ResourceRequestProto resourceRequestProto) { final Resource result = Records.newRecord(Resource.class); final int memory = getMemory(resourceRequestProto.getMemorySize()); final int core = resourceRequestProto.getVirtualCores(); LOG.log(Level.FINEST, "Resource requested: memory = {0}, virtual core count = {1}.", new Object[] { memory, core }); result.setMemory(memory);/* w ww. j ava 2 s . co m*/ result.setVirtualCores(core); return result; }
From source file:org.apache.slider.providers.slideram.SliderAMClientProvider.java
License:Apache License
/** * Update the AM resource with any local needs * @param capability capability to update *//*w w w. j a v a 2 s . com*/ public void prepareAMResourceRequirements(MapOperations sliderAM, Resource capability) { capability.setMemory(sliderAM.getOptionInt(ResourceKeys.YARN_MEMORY, capability.getMemory())); capability.setVirtualCores(sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); }
From source file:org.apache.slider.server.appmaster.state.RoleStatus.java
License:Apache License
/** * Given a resource, set its requirements to those this role needs * @param resource resource to configure * @return the resource/*w w w .j av a2 s . c o m*/ */ public Resource copyResourceRequirements(Resource resource) { Preconditions.checkNotNull(resourceRequirements, "Role resource requirements have not been set"); resource.setMemory(resourceRequirements.getMemory()); resource.setVirtualCores(resourceRequirements.getVirtualCores()); return resource; }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress) throws IOException, YarnException { QueryId queryId = queryInProgress.getQueryId(); ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId); LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" + queryId + ", appId=" + appId); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId);/*from w w w. ja v a 2 s . c om*/ // set the application name appContext.setApplicationName("Tajo"); Priority pri = Records.newRecord(Priority.class); pri.setPriority(5); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); ContainerLaunchContext commonContainerLaunchContext = YarnContainerProxy .createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true); // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m"); for (String eachToken : jvmOptions.split((" "))) { vargs.add(eachToken); } // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("qm"); vargs.add(queryId.toString()); // queryId vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":" + masterContext.getTajoMasterService().getBindAddress().getPort()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up QueryMasterRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); final Resource resource = Records.newRecord(Resource.class); // TODO - get default value from conf resource.setMemory(2000); resource.setVirtualCores(1); Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext( commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>(2)); appContext.setAMContainerSpec(masterContainerContext); LOG.info("Submitting QueryMaster to ResourceManager"); yarnClient.submitApplication(appContext); ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching QueryMaster with appAttemptId: " + attemptId); return attemptId; }