List of usage examples for org.apache.hadoop.yarn.api.records Priority setPriority
@Public @Stable public abstract void setPriority(int priority);
From source file:oz.hadoop.yarn.api.core.ApplicationMasterLauncherImpl.java
License:Apache License
/** * *///from w w w . j a v a 2 s . c o m private ApplicationSubmissionContext initApplicationContext(YarnClientApplication yarnClientApplication) { ApplicationSubmissionContext appContext = yarnClientApplication.getApplicationSubmissionContext(); appContext.setApplicationName(this.applicationName); this.applicationId = appContext.getApplicationId(); this.applicationSpecification.put(YayaConstants.APP_ID, this.applicationId.getId()); ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); Map<String, LocalResource> localResources = this.createLocalResources(); if (logger.isDebugEnabled()) { logger.debug("Created LocalResources: " + localResources); } containerLaunchContext.setLocalResources(localResources); String jsonArguments = JSONObject.toJSONString(this.applicationSpecification); String encodedJsonArguments = new String(Base64.encodeBase64(jsonArguments.getBytes())); YayaUtils.inJvmPrep("JAVA", containerLaunchContext, AM_CLASS_NAME, encodedJsonArguments); String applicationMasterLaunchCommand = this.createApplicationMasterLaunchCommand(localResources, encodedJsonArguments); containerLaunchContext.setCommands(Collections.singletonList(applicationMasterLaunchCommand)); Priority priority = Records.newRecord(Priority.class); priority.setPriority((int) this.applicationSpecification.get(YayaConstants.PRIORITY)); Resource capability = Records.newRecord(Resource.class); capability.setMemory((int) this.applicationSpecification.get(YayaConstants.MEMORY)); capability.setVirtualCores((int) this.applicationSpecification.get(YayaConstants.VIRTUAL_CORES)); appContext.setResource(capability); appContext.setMaxAppAttempts((int) this.applicationSpecification.get(YayaConstants.MAX_ATTEMPTS)); appContext.setAMContainerSpec(containerLaunchContext); appContext.setPriority(priority); appContext.setQueue((String) this.applicationSpecification.get(YayaConstants.QUEUE_NAME)); if (logger.isDebugEnabled()) { logger.debug("Created ApplicationSubmissionContext: " + appContext); } return appContext; }
From source file:proxyyarn.ProxyYarn.java
License:Apache License
public boolean run() throws Exception { Configuration conf = new YarnConfiguration(new Configuration()); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf);/*from w ww.ja v a2s . c o m*/ yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); log.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); log.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { log.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo("default"); log.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { log.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } FileSystem fs = FileSystem.get(conf); if (!fs.getClass().equals(DistributedFileSystem.class)) { log.error("Expected DistributedFileSystem, but was {}", fs.getClass().getSimpleName()); System.exit(1); } // ApplicationClientProtocol applicationsManager; // InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); // log.info("Connecting to ResourceManager at {}", rmAddress); // Configuration appManagerServerConf = new Configuration(conf); // YarnRPC rpc = YarnRPC.create(appManagerServerConf); // ApplicationClientProtocol applicationManager = (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class, rmAddress, appManagerServerConf); String appName = "AccumuloProxyYarn"; YarnClientApplication app = yarnClient.createApplication(); // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class); // GetNewApplicationResponse response = applicationManager.getNewApplication(request); // log.info("Got new ApplicationId=" + response.getApplicationId()); // ApplicationId appId = response.getApplicationId(); // Create a new ApplicationSubmissionContext // ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the ApplicationId // appContext.setApplicationId(appId); // set the application name // appContext.setApplicationName(appName); // Create a new container launch context for the AM's container ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // Define the local resources required Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // Lets assume the jar we need for our ApplicationMaster is available in // HDFS at a certain known path to us and we want to make it available to // the ApplicationMaster in the launched container Path localJarPath = new Path( "file:///Users/jelser/projects/accumulo-proxy-yarn/target/accumulo-proxy-yarn-0.0.1-SNAPSHOT.jar"); Path jarPath = new Path("hdfs:///accumulo-proxy-yarn-0.0.1-SNAPSHOT.jar"); fs.copyFromLocalFile(false, true, localJarPath, jarPath); FileStatus jarStatus = fs.getFileStatus(jarPath); LocalResource amJarRsrc = Records.newRecord(LocalResource.class); // Set the type of resource - file or archive // archives are untarred at the destination by the framework amJarRsrc.setType(LocalResourceType.FILE); // Set visibility of the resource // Setting to most private option i.e. this file will only // be visible to this instance of the running application amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION); // Set the location of resource to be copied over into the // working directory amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(jarPath)); // Set timestamp and length of file so that the framework // can do basic sanity checks for the local resource // after it has been copied over to ensure it is the same // resource the client intended to use with the application amJarRsrc.setTimestamp(jarStatus.getModificationTime()); amJarRsrc.setSize(jarStatus.getLen()); // The framework will create a symlink called AppMaster.jar in the // working directory that will be linked back to the actual file. // The ApplicationMaster, if needs to reference the jar file, would // need to use the symlink filename. localResources.put("AppMaster.jar", amJarRsrc); // Set the local resources into the launch context amContainer.setLocalResources(localResources); // Set up the environment needed for the launch context Map<String, String> env = new HashMap<String, String>(); // For example, we could setup the classpath needed. // Assuming our classes or jars are available as local resources in the // working directory from which the command will be run, we need to append // "." to the path. // By default, all the hadoop specific classpaths will already be available // in $CLASSPATH, so we should be careful not to overwrite it. String classPathEnv = "$CLASSPATH:./*:/Users/jelser/projects/accumulo-proxy-yarn/target/lib/*"; env.put("CLASSPATH", classPathEnv); amContainer.setEnvironment(env); // Construct the command to be executed on the launched container String command = "${JAVA_HOME}" + "/bin/java" + " proxyyarn.ProxyYarnAppMaster 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; List<String> commands = new ArrayList<String>(); commands.add(command); // add additional commands if needed // Set the command array into the container spec amContainer.setCommands(commands); // Define the resource requirements for the container // For now, YARN only supports memory so we set the memory // requirements. // If the process takes more than its allocated memory, it will // be killed by the framework. // Memory being requested for should be less than max capability // of the cluster and all asks should be a multiple of the min capability. Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); appContext.setResource(capability); // Create the request to send to the ApplicationsManager // SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); // appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the ApplicationsManager // Ignore the response as either a valid response object is returned on // success or an exception thrown to denote the failure // applicationManager.submitApplication(appRequest); // Set the container launch content into the ApplicationSubmissionContext appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure log.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); return monitorApplication(yarnClient, appId); /* Thread.sleep(200); boolean running = false; while(true) { GetApplicationReportRequest reportRequest = Records.newRecord(GetApplicationReportRequest.class); reportRequest.setApplicationId(appId); GetApplicationReportResponse reportResponse = applicationManager.getApplicationReport(reportRequest); ApplicationReport report = reportResponse.getApplicationReport(); log.info(report.toString()); YarnApplicationState state = report.getYarnApplicationState(); switch (state) { case NEW: case NEW_SAVING: case SUBMITTED: case ACCEPTED: log.info("State: {}", state); break; case RUNNING: log.info("Running application"); running = true; break; case FINISHED: case FAILED: case KILLED: log.info("State: {}", state); return; default: log.info("Unknown state: {}", state); return; } if (!running) { Thread.sleep(1000); } }*/ }
From source file:proxyyarn.ProxyYarnAppMaster.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * //from w w w . j a va 2s. c o m * @return the setup ResourceRequest to be sent to RM */ private static ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(1); // Set up resource type requirements // For now, only memory is supported so we set memory requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); ContainerRequest request = new ContainerRequest(capability, null, null, pri); log.info("Requested container ask: " + request.toString()); return request; }
From source file:runtime.starter.MPJAppMaster.java
License:Open Source License
public void run() throws Exception { try {/*from w ww.j a v a 2s. co m*/ appMasterSock = new Socket(serverName, ioServerPort); //redirecting stdout and stderr System.setOut(new PrintStream(appMasterSock.getOutputStream(), true)); System.setErr(new PrintStream(appMasterSock.getOutputStream(), true)); } catch (Exception exp) { exp.printStackTrace(); } FileSystem fs = FileSystem.get(conf); Path wrapperDest = new Path(wrapperPath); FileStatus destStatus = fs.getFileStatus(wrapperDest); Path userFileDest = new Path(userJarPath); FileStatus destStatusClass = fs.getFileStatus(userFileDest); // Initialize AM <--> RM communication protocol AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); // Initialize AM <--> NM communication protocol NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager RegisterApplicationMasterResponse registerResponse = rmClient.registerApplicationMaster("", 0, ""); // Priority for containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(mpjContainerPriority); maxMem = registerResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { System.out.println("[MPJAppMaster]: Max memory capability resources " + "in cluster: " + maxMem); } if (containerMem > maxMem) { System.out.println("[MPJAppMaster]: container memory specified above " + "threshold of cluster! Using maximum memory for " + "containers: " + containerMem); containerMem = maxMem; } maxCores = registerResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { System.out.println("[MPJAppMaster]: Max v-cores capability resources " + "in cluster: " + maxCores); } if (containerCores > maxCores) { System.out.println("[MPJAppMaster]: virtual cores specified above " + "threshold of cluster! Using maximum v-cores for " + "containers: " + containerCores); containerCores = maxCores; } // Resource requirements for containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMem); capability.setVirtualCores(containerCores); // Make container requests to ResourceManager for (int i = 0; i < np; ++i) { ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerReq); } Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // Creating Local Resource for Wrapper LocalResource wrapperJar = Records.newRecord(LocalResource.class); wrapperJar.setResource(ConverterUtils.getYarnUrlFromPath(wrapperDest)); wrapperJar.setSize(destStatus.getLen()); wrapperJar.setTimestamp(destStatus.getModificationTime()); wrapperJar.setType(LocalResourceType.ARCHIVE); wrapperJar.setVisibility(LocalResourceVisibility.APPLICATION); // Creating Local Resource for UserClass LocalResource userClass = Records.newRecord(LocalResource.class); userClass.setResource(ConverterUtils.getYarnUrlFromPath(userFileDest)); userClass.setSize(destStatusClass.getLen()); userClass.setTimestamp(destStatusClass.getModificationTime()); userClass.setType(LocalResourceType.ARCHIVE); userClass.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put("mpj-yarn-wrapper.jar", wrapperJar); localResources.put("user-code.jar", userClass); while (allocatedContainers < np) { AllocateResponse response = rmClient.allocate(0); mpiContainers.addAll(response.getAllocatedContainers()); allocatedContainers = mpiContainers.size(); if (allocatedContainers != np) { Thread.sleep(100); } } if (debugYarn) { System.out.println("[MPJAppMaster]: launching " + allocatedContainers + " containers"); } for (Container container : mpiContainers) { ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add(" $JAVA_HOME/bin/java"); commands.add(" -Xmx" + containerMem + "m"); commands.add(" runtime.starter.MPJYarnWrapper"); commands.add("--serverName"); commands.add(serverName); // server name commands.add("--ioServerPort"); commands.add(Integer.toString(ioServerPort)); // IO server port commands.add("--deviceName"); commands.add(deviceName); // device name commands.add("--className"); commands.add(className); // class name commands.add("--psl"); commands.add(psl); // protocol switch limit commands.add("--np"); commands.add(Integer.toString(np)); // no. of containers commands.add("--rank"); commands.add(" " + Integer.toString(rank++)); // rank //temp sock port to share rank and ports commands.add("--wireUpPort"); commands.add(wireUpPort); if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } ctx.setCommands(commands); // Set local resource for containers ctx.setLocalResources(localResources); // Set environment for container Map<String, String> containerEnv = new HashMap<String, String>(); setupEnv(containerEnv); ctx.setEnvironment(containerEnv); // Time to start the container nmClient.startContainer(container, ctx); } while (completedContainers < np) { // argument to allocate() is the progress indicator AllocateResponse response = rmClient.allocate(completedContainers / np); for (ContainerStatus status : response.getCompletedContainersStatuses()) { if (debugYarn) { System.out.println("\n[MPJAppMaster]: Container Id - " + status.getContainerId()); System.out.println("[MPJAppMaster]: Container State - " + status.getState().toString()); System.out.println("[MPJAppMaster]: Container Diagnostics - " + status.getDiagnostics()); } ++completedContainers; } if (completedContainers != np) { Thread.sleep(100); } ; } // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); //shutDown AppMaster IO System.out.println("EXIT"); }
From source file:tachyon.yarn.ApplicationMaster.java
License:Apache License
public void requestContainers() throws Exception { // Priority for Tachyon master and worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for master containers Resource masterResource = Records.newRecord(Resource.class); masterResource.setMemory(mMasterMemInMB); masterResource.setVirtualCores(mMasterCpu); String[] nodes = { mMasterAddress }; // Make container request for Tachyon master to ResourceManager boolean relaxLocality = true; if (!mMasterAddress.equals("localhost")) { relaxLocality = false;/*from w w w. jav a2 s . co m*/ } ContainerRequest masterContainerAsk = new ContainerRequest(masterResource, nodes, null /* any racks */, priority, relaxLocality); LOG.info("Making resource request for Tachyon master: cpu {} memory {} MB on node {}", masterResource.getVirtualCores(), masterResource.getMemory(), mMasterAddress); mRMClient.addContainerRequest(masterContainerAsk); // Wait until Tachyon master container has been allocated while (!mMasterContainerAllocated) { Thread.sleep(1000); } // Resource requirements for master containers Resource workerResource = Records.newRecord(Resource.class); workerResource.setMemory(mWorkerMemInMB + mRamdiskMemInMB); workerResource.setVirtualCores(mWorkerCpu); // Make container requests for workers to ResourceManager for (int i = 0; i < mNumWorkers; i++) { ContainerRequest containerAsk = new ContainerRequest(workerResource, null /* any hosts */, null /* any racks */, priority); LOG.info("Making resource request for Tachyon worker {}: cpu {} memory {} MB on any nodes", i, workerResource.getVirtualCores(), workerResource.getMemory()); mRMClient.addContainerRequest(containerAsk); } // Wait until all Tachyon worker containers have been allocated while (mNumAllocatedWorkerContainers < mNumWorkers) { Thread.sleep(1000); } LOG.info("Master and workers are launched"); // Wait for 5 more seconds to avoid application unregistered before some container fully // launched. while (!mApplicationDone) { Thread.sleep(5000); } }
From source file:yarnkit.container.ContainerLaunchContextFactory.java
License:Apache License
@Nonnull public Priority createPriority(@Nonnull ContainerLaunchParameters parameters) { int priority = parameters.getPriority(); Priority p = Records.newRecord(Priority.class); p.setPriority(priority); return p;/*from ww w.j a va 2 s . com*/ }
From source file:yrun.YarnRunnerApplicationMaster.java
License:Apache License
public void run(String[] args) throws Exception { JsonParser parser = new JsonParser(); JsonElement element = parser.parse(new FileReader(YarnRunner.MASTER_JSON)); LOG.info("Json [" + element + "]"); JsonObject jsonObject = (JsonObject) element; int priority = jsonObject.get("priority").getAsInt(); int numberOfContainers = jsonObject.get("numberOfContainers").getAsInt(); int memory = jsonObject.get("memory").getAsInt(); int vCores = jsonObject.get("vCores").getAsInt(); String command = jsonObject.get("command").getAsString(); // startHttpServer(); // InetSocketAddress address = _server.getAddress(); // LOG.info("Http server started at [" + address + "]"); // String appHostName = "app-host-name"; // int appHostPort = address.getPort(); // String appTrackingUrl = "http://" + address.getHostName() + ":" + // appHostPort + "/"; // LOG.info("App Tracking Url [" + appTrackingUrl + "]"); // Initialize clients to ResourceManager and NodeManagers Configuration conf = new YarnConfiguration(); AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf);/*w ww . j a v a2s. co m*/ rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager LOG.info("Register Application Master 0"); String appHostName = ""; int appHostPort = 0; String appTrackingUrl = ""; rmClient.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl); LOG.info("Register Application Master 1"); // Priority for worker containers - priorities are intra-application Priority priorityRecord = Records.newRecord(Priority.class); priorityRecord.setPriority(priority); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); capability.setVirtualCores(vCores); // Make container requests to ResourceManager for (int i = 0; i < numberOfContainers; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priorityRecord); LOG.info("Making resource request for [" + i + "]"); rmClient.addContainerRequest(containerAsk); } // Obtain allocated containers, launch and check for responses int responseId = 0; int completedContainers = 0; long lastReport = 0; List<Container> containers = new ArrayList<Container>(); while (completedContainers < numberOfContainers) { if (lastReport + TimeUnit.SECONDS.toNanos(5) < System.nanoTime()) { for (Container container : containers) { ContainerId containerId = container.getId(); NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = nmClient.getContainerStatus(containerId, nodeId); LOG.info("NodeId [" + nodeId + "] Container Status [" + containerStatus + "]"); // Figure out } lastReport = System.nanoTime(); } AllocateResponse response = rmClient.allocate(responseId++); for (Container container : response.getAllocatedContainers()) { containers.add(container); // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setCommands( Collections.singletonList(command + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout2" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); LOG.info("Launching container " + container.getId()); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { completedContainers++; LOG.info("Completed container " + status.getContainerId()); } Thread.sleep(100); } // _server.stop(0); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }