List of usage examples for org.apache.hadoop.yarn.api.records Resource newInstance
@Public
@Stable
public static Resource newInstance(long memory, int vCores)
From source file:com.splicemachine.yarn.test.BareYarnTest.java
License:Apache License
/** * All we really need to do here is to create a yarn client, configure it using the same * yarn-site.xml as was used by the server to start up. * @throws YarnException// w w w . ja v a 2 s .c o m * @throws IOException */ @Test(timeout = 60000) @Ignore("Broken by dependency change") public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException { // create, submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( Collections.<String, LocalResource>emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>()); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start RMAppAttempt appAttempt; while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnPlatform.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser( UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); }
From source file:de.huberlin.wbi.hiway.am.benchmark.PerfectDaxGreedyQueue.java
License:Apache License
@Override public void enqueueResourceRequest(TaskInstance task) { DaxTaskInstance daxTask = (DaxTaskInstance) task; // request a little more to be safe int memoryMB = Math.min(20000, (int) (daxTask.getPeakMemoryBytes() / 1000000 + 200)); Priority pri = Priority.newInstance(requestPriority); Resource capability = Resource.newInstance(memoryMB, 1); AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(capability, new String[0], null, pri, getRelaxLocality()); unissuedContainerRequests.add(containerRequest); // remember when the container request was published requestPublishedTime.put(containerRequest.getCapability().getMemory(), System.currentTimeMillis()); WorkflowDriver.Logger.writeToStdout(String.format("PerfDaxGQ requested %s MB for task %s", containerRequest.getCapability().getMemory(), task.getTaskName())); queue.add(daxTask);/* www .j ava2 s . co m*/ /* log */ WorkflowDriver.Logger.writeToStdout("Added task " + task + " to queue"); }
From source file:de.huberlin.wbi.hiway.scheduler.ma.MemoryAware.java
License:Apache License
@SuppressWarnings("unchecked") @Override//from ww w . j a va2 s . com public boolean hasNextNodeRequest() { boolean hasNextNodeRequest = super.hasNextNodeRequest(); // Hadoop has a weird bug where open container requests are sometimes "forgotten" (i.e., not fulfilled despite available resources). // Running into this bug can be prevented by occasionally re-issuing ContainerRequests. if (!hasNextNodeRequest) { List<? extends Collection<ContainerRequest>> requestCollections = amRMClient.getMatchingRequests( Priority.newInstance(requestPriority), ResourceRequest.ANY, Resource.newInstance(maxMem, maxCores)); for (Collection<ContainerRequest> requestCollection : requestCollections) { ContainerRequest first = requestCollection.iterator().next(); amRMClient.removeContainerRequest(first); amRMClient.addContainerRequest(first); } } return hasNextNodeRequest; }
From source file:de.huberlin.wbi.hiway.scheduler.WorkflowScheduler.java
License:Apache License
/** * Create a request for resources.// ww w.j a v a2 s .c om * @param nodes Must be located on one of the given nodes, or pass empty array. */ protected ContainerRequest setupContainerAskForRM(String[] nodes, int memoryMegaBytes) { // set the priority for the request Priority pri = Priority.newInstance(requestPriority); // pri.setPriority(requestPriority); // set up resource type requirements Resource capability = Resource.newInstance(memoryMegaBytes, containerCores); // capability.setMemory(containerMemoryMegaBytes); // capability.setVirtualCores(containerCores); return new ContainerRequest(capability, nodes, null, pri, getRelaxLocality()); }
From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException//from ww w.ja v a 2 s . c om * @throws IOException */ @SuppressWarnings({ "unchecked" }) public boolean run() throws YarnException, IOException { yarnClient.start(); LOG.info("Starting ApplicationMaster"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amRMClient = AMRMClient.createAMRMClient(); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); //TODO: Figure out how to do this. List<NodeReport> reports = this.yarnClient.getNodeReports(); LOG.info("Cluster Status"); List<Resource> availableResources = new ArrayList<Resource>(); for (NodeReport nr : reports) { LOG.info(" NodeId: " + nr.getNodeId() + " Capabilities " + nr.getCapability() + " Used Resources " + nr.getUsed()); int availableMem = nr.getCapability().getMemory() - nr.getUsed().getMemory(); int availableVCores = nr.getCapability().getVirtualCores() - nr.getUsed().getVirtualCores(); Resource resource = Resource.newInstance(availableMem, availableVCores); availableResources.add(resource); } /*Based on available resources scheduler should decide the best allocation of recommenders to resources and return a list of resources that should be requested from the ResourceManager*/ DataSetDescription datasetDesc = new DataSetDescription(this.setup.dataMetadataFile); List<RecommenderPool> recommenderPools = RecommenderScheduler.splitRecommenderPool(availableResources, recommenders, datasetDesc, this.setup.nShards); for (RecommenderPool res : recommenderPools) { ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority); LOG.info("CONTAINER ASK: " + containerAsk); amRMClient.addContainerRequest(containerAsk); } float progress = 0; List<RecommenderPool> pendingPools = new ArrayList<RecommenderPool>(); for (RecommenderPool p : recommenderPools) pendingPools.add(p); this.numTotalContainers = recommenderPools.size(); this.numCompletedContainers.set(0); this.numAllocatedContainers.set(0); while (numCompletedContainers.get() != numTotalContainers) { try { Thread.sleep(200); AllocateResponse allocResp = amRMClient.allocate(progress); List<Container> newContainers = allocResp.getAllocatedContainers(); List<ContainerStatus> completedContainers = allocResp.getCompletedContainersStatuses(); if (this.numAllocatedContainers.get() >= this.numTotalContainers && pendingPools.size() != 0) { //Ask for new containers for pending pools LOG.warn("The number of allocated containers has exceeded number of total containers, but " + "the pending pool size is still not 0. Asking for new containers from RM"); for (RecommenderPool res : pendingPools) { ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority); LOG.info("NEW CONTAINER ASK: " + containerAsk); amRMClient.addContainerRequest(containerAsk); } } if (newContainers.size() > 0) { LOG.info("Allocated " + newContainers.size() + " new containers"); numAllocatedContainers.addAndGet(newContainers.size()); for (Container container : newContainers) { //Find matching recommender pool from pendingRecommender pools. RecommenderPool pool = null; for (RecommenderPool p : pendingPools) { if (p.getTotalMemory() == container.getResource().getMemory()) { pool = p; break; } } if (pool == null) { LOG.warn("No Takers for Container " + container + " Releasing container"); amRMClient.releaseAssignedContainer(container.getId()); } else { startContainer(container, pool); //This pool has now got a container. Remove it from pending pools pendingPools.remove(pool); } } } onContainersCompleted(completedContainers); } catch (InterruptedException ex) { } } finish(); return success; }
From source file:gobblin.yarn.GobblinYarnAppLauncher.java
License:Apache License
private Resource prepareContainerResource(GetNewApplicationResponse newApplicationResponse) { int memoryMbs = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_MEMORY_MBS_KEY); int maximumMemoryCapacity = newApplicationResponse.getMaximumResourceCapability().getMemory(); if (memoryMbs > maximumMemoryCapacity) { LOGGER.info(String.format( "Specified AM memory [%d] is above the maximum memory capacity [%d] of the " + "cluster, using the maximum memory capacity instead.", memoryMbs, maximumMemoryCapacity)); memoryMbs = maximumMemoryCapacity; }//from ww w . j a v a2 s. c o m int vCores = this.config.getInt(GobblinYarnConfigurationKeys.APP_MASTER_CORES_KEY); int maximumVirtualCoreCapacity = newApplicationResponse.getMaximumResourceCapability().getVirtualCores(); if (vCores > maximumVirtualCoreCapacity) { LOGGER.info(String.format( "Specified AM vcores [%d] is above the maximum vcore capacity [%d] of the " + "cluster, using the maximum vcore capacity instead.", memoryMbs, maximumMemoryCapacity)); vCores = maximumVirtualCoreCapacity; } // Set up resource type requirements for ApplicationMaster return Resource.newInstance(memoryMbs, vCores); }
From source file:io.amient.yarn1.YarnContainerContext.java
License:Open Source License
public YarnContainerContext(Configuration yarnConfig, Properties appConfig, String jvmArgs, int priority, int memoryMb, int numCores, String applicationName, Class<?> mainClass, String[] args) throws Exception { this.appConfig = appConfig; this.yarnConfig = yarnConfig; this.jvmArgs = jvmArgs; this.yarnConfig.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); this.yarnConfig.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); this.jarName = applicationName + ".jar"; this.mainClass = mainClass; this.mainClassName = mainClass.getName().replace("$", ""); this.capability = Resource.newInstance(memoryMb, numCores); this.priority = Priority.newInstance(priority); this.args = args; }
From source file:io.hops.metadata.util.DistributedRTClientEvaluation.java
License:Apache License
/** * Registers a node with the RT. If num is greater that 1, multiple requests * are sent for the same node and the last response is returned; * <p/>//w ww. ja v a2s .co m * * @param rt * @param host * @param port * @param num * @return */ private void registerClient(ResourceTracker rt, NodeId nodeId) { try { RegisterNodeManagerRequest request = Records.newRecord(RegisterNodeManagerRequest.class); request.setHttpPort(nodeId.getPort()); request.setNodeId(nodeId); Resource resource = Resource.newInstance(5012, 8); request.setResource(resource); rt.registerNodeManager(request); } catch (YarnException ex) { LOG.error("HOP :: Error sending NodeHeartbeatResponse", ex); } catch (IOException ex) { LOG.error("HOP :: Error sending NodeHeartbeatResponse", ex); } }
From source file:io.hops.tensorflow.ApplicationMaster.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM *///w w w . j av a 2s. c o m public ContainerRequest setupContainerAskForRM(boolean worker) { Priority pri = Priority.newInstance(1); // Set up resource type requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); if (worker) { pri.setPriority(0); // worker: 0, ps: 1 capability.setGPUs(containerGPUs); } ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
From source file:io.hops.tensorflow.Client.java
License:Apache License
private ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication app, ContainerLaunchContext containerContext) { ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); if (name == null) { appContext.setApplicationName(mainRelativePath); } else {//from w ww . j ava 2 s . c o m appContext.setApplicationName(name); } appContext.setApplicationType("YARNTF"); appContext.setMaxAppAttempts(maxAppAttempts); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } appContext.setResource(Resource.newInstance(amMemory, amVCores)); appContext.setAMContainerSpec(containerContext); appContext.setPriority(Priority.newInstance(amPriority)); appContext.setQueue(amQueue); // the queue to which this application is to be submitted in the RM return appContext; }