List of usage examples for org.apache.hadoop.yarn.api.records Resource getMemory
@Public @Deprecated public abstract int getMemory();
From source file:org.apache.tez.dag.app.rm.LlapTaskSchedulerService.java
License:Apache License
@Override public Resource getTotalResources() { int memory = 0; int vcores = 0; readLock.lock();/*from ww w. j a va2 s.c o m*/ try { for (ServiceInstance inst : activeInstances.getAll().values()) { if (inst.isAlive()) { Resource r = inst.getResource(); LOG.info("Found instance " + inst); memory += r.getMemory(); vcores += r.getVirtualCores(); } else { LOG.info("Ignoring dead instance " + inst); } } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }
From source file:org.apache.tez.dag.app.rm.LlapTaskSchedulerService.java
License:Apache License
/** * The difference between this and getTotalResources() is that this only gives currently free * resource instances, while the other lists all the instances that may become available in a * while.//from w w w . j av a 2s .c o m */ @Override public Resource getAvailableResources() { // need a state store eventually for current state & measure backoffs int memory = 0; int vcores = 0; readLock.lock(); try { for (Entry<ServiceInstance, NodeInfo> entry : instanceToNodeMap.entrySet()) { if (entry.getKey().isAlive() && !entry.getValue().isDisabled()) { Resource r = entry.getKey().getResource(); memory += r.getMemory(); vcores += r.getVirtualCores(); } } } finally { readLock.unlock(); } return Resource.newInstance(memory, vcores); }
From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
synchronized void preemptIfNeeded() { Resource freeResources = Resources.subtract(totalResources, allocatedResources); LOG.info("Allocated resource memory: " + allocatedResources.getMemory() + " cpu:" + allocatedResources.getVirtualCores()); assert freeResources.getMemory() >= 0; CookieContainerRequest highestPriRequest = null; for (CookieContainerRequest request : taskRequests.values()) { if (highestPriRequest == null) { highestPriRequest = request; } else if (isHigherPriority(request.getPriority(), highestPriRequest.getPriority())) { highestPriRequest = request; }//from w w w . ja v a 2 s.c om } if (highestPriRequest != null && !fitsIn(highestPriRequest.getCapability(), freeResources)) { // highest priority request will not fit in existing free resources // free up some more // TODO this is subject to error wrt RM resource normalization Map.Entry<Object, Container> preemptedEntry = null; for (Map.Entry<Object, Container> entry : taskAllocations.entrySet()) { if (!isHigherPriority(highestPriRequest.getPriority(), entry.getValue().getPriority())) { // higher or same priority continue; } if (preemptedEntry == null || !isHigherPriority(entry.getValue().getPriority(), preemptedEntry.getValue().getPriority())) { // keep the lower priority or the one added later preemptedEntry = entry; } } if (preemptedEntry != null) { // found something to preempt LOG.info("Preempting task: " + preemptedEntry.getKey() + " to free resource for request: " + highestPriRequest + " . Current free resources: " + freeResources); deallocateContainer(preemptedEntry.getValue().getId()); // app client will be notified when after container is killed // and we get its completed container status } } }
From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
private boolean fitsIn(Resource toFit, Resource resource) { // YARN-893 prevents using correct library code //return Resources.fitsIn(toFit, resource); return resource.getMemory() >= toFit.getMemory(); }
From source file:org.apache.tez.dag.app.rm.TestLocalTaskSchedulerService.java
License:Apache License
@Test(timeout = 5000) public void testCreateResource() { Resource resource; //value in integer long value = 4 * 1024 * 1024; resource = ltss.createResource(value, core); Assert.assertEquals((int) (value / (1024 * 1024)), resource.getMemory()); }
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
boolean canFit(Resource arg0, Resource arg1) { int mem0 = arg0.getMemory(); int mem1 = arg1.getMemory(); int cpu0 = arg0.getVirtualCores(); int cpu1 = arg1.getVirtualCores(); if (mem0 <= mem1 && cpu0 <= cpu1) { return true; }/*w ww . j av a2s . c om*/ return false; }
From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java
License:Apache License
void preemptIfNeeded() { if (preemptionPercentage == 0) { // turned off return;//from w w w .jav a2 s . c o m } ContainerId[] preemptedContainers = null; int numPendingRequestsToService = 0; synchronized (this) { Resource freeResources = amRmClient.getAvailableResources(); if (LOG.isDebugEnabled()) { LOG.debug("Allocated resource memory: " + allocatedResources.getMemory() + " cpu:" + allocatedResources.getVirtualCores() + " delayedContainers: " + delayedContainerManager.delayedContainers.size() + " heartbeats: " + numHeartbeats + " lastPreemptionHeartbeat: " + heartbeatAtLastPreemption); } assert freeResources.getMemory() >= 0; CookieContainerRequest highestPriRequest = null; int numHighestPriRequests = 0; for (CookieContainerRequest request : taskRequests.values()) { if (highestPriRequest == null) { highestPriRequest = request; numHighestPriRequests = 1; } else if (isHigherPriority(request.getPriority(), highestPriRequest.getPriority())) { highestPriRequest = request; numHighestPriRequests = 1; } else if (request.getPriority().equals(highestPriRequest.getPriority())) { numHighestPriRequests++; } } if (highestPriRequest == null) { // nothing pending return; } if (fitsIn(highestPriRequest.getCapability(), freeResources)) { if (LOG.isDebugEnabled()) { LOG.debug("Highest pri request: " + highestPriRequest + " fits in available resources " + freeResources); } return; } // highest priority request will not fit in existing free resources // free up some more // TODO this is subject to error wrt RM resource normalization numPendingRequestsToService = scaleDownByPreemptionPercentage(numHighestPriRequests, preemptionPercentage); if (numPendingRequestsToService < 1) { return; } if (LOG.isDebugEnabled()) { LOG.debug("Trying to service " + numPendingRequestsToService + " out of total " + numHighestPriRequests + " pending requests at pri: " + highestPriRequest.getPriority()); } for (int i = 0; i < numPendingRequestsToService; ++i) { // This request must have been considered for matching with all existing // containers when request was made. Container lowestPriNewContainer = null; // could not find anything to preempt. Check if we can release unused // containers for (HeldContainer heldContainer : delayedContainerManager.delayedContainers) { if (!heldContainer.isNew()) { if (LOG.isDebugEnabled()) { LOG.debug("Reused container exists. Wait for assignment loop to release it. " + heldContainer.getContainer().getId()); } return; } if (heldContainer.geNumAssignmentAttempts() < 3) { // we havent tried to assign this container at node/rack/ANY if (LOG.isDebugEnabled()) { LOG.debug("Brand new container. Wait for assignment loop to match it. " + heldContainer.getContainer().getId()); } return; } Container container = heldContainer.getContainer(); if (lowestPriNewContainer == null || isHigherPriority(lowestPriNewContainer.getPriority(), container.getPriority())) { // there is a lower priority new container lowestPriNewContainer = container; } } if (lowestPriNewContainer != null) { LOG.info("Preempting new container: " + lowestPriNewContainer.getId() + " with priority: " + lowestPriNewContainer.getPriority() + " to free resource for request: " + highestPriRequest + " . Current free resources: " + freeResources); numPendingRequestsToService--; releaseUnassignedContainers(Collections.singletonList(lowestPriNewContainer)); // We are returning an unused resource back the RM. The RM thinks it // has serviced our initial request and will not re-allocate this back // to us anymore. So we need to ask for this again. If there is no // outstanding request at that priority then its fine to not ask again. // See TEZ-915 for more details for (Map.Entry<Object, CookieContainerRequest> entry : taskRequests.entrySet()) { Object task = entry.getKey(); CookieContainerRequest request = entry.getValue(); if (request.getPriority().equals(lowestPriNewContainer.getPriority())) { LOG.info("Resending request for task again: " + task); deallocateTask(task, true); allocateTask(task, request.getCapability(), (request.getNodes() == null ? null : request.getNodes().toArray(new String[request.getNodes().size()])), (request.getRacks() == null ? null : request.getRacks().toArray(new String[request.getRacks().size()])), request.getPriority(), request.getCookie().getContainerSignature(), request.getCookie().getAppCookie()); break; } } // come back and free more new containers if needed continue; } } if (numPendingRequestsToService < 1) { return; } // there are no reused or new containers to release. try to preempt running containers // this assert will be a no-op in production but can help identify // invalid assumptions during testing assert delayedContainerManager.delayedContainers.isEmpty(); if ((numHeartbeats - heartbeatAtLastPreemption) <= numHeartbeatsBetweenPreemptions) { return; } Priority preemptedTaskPriority = null; int numEntriesAtPreemptedPriority = 0; for (Map.Entry<Object, Container> entry : taskAllocations.entrySet()) { HeldContainer heldContainer = heldContainers.get(entry.getValue().getId()); CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo(); Priority taskPriority = lastTaskInfo.getPriority(); Object signature = lastTaskInfo.getCookie().getContainerSignature(); if (!isHigherPriority(highestPriRequest.getPriority(), taskPriority)) { // higher or same priority continue; } if (containerSignatureMatcher.isExactMatch(highestPriRequest.getCookie().getContainerSignature(), signature)) { // exact match with different priorities continue; } if (preemptedTaskPriority == null || !isHigherPriority(taskPriority, preemptedTaskPriority)) { // keep the lower priority if (taskPriority.equals(preemptedTaskPriority)) { numEntriesAtPreemptedPriority++; } else { // this is at a lower priority than existing numEntriesAtPreemptedPriority = 1; } preemptedTaskPriority = taskPriority; } } if (preemptedTaskPriority != null) { int newNumPendingRequestsToService = scaleDownByPreemptionPercentage( Math.min(numEntriesAtPreemptedPriority, numHighestPriRequests), preemptionPercentage); numPendingRequestsToService = Math.min(newNumPendingRequestsToService, numPendingRequestsToService); if (numPendingRequestsToService < 1) { return; } LOG.info("Trying to service " + numPendingRequestsToService + " out of total " + numHighestPriRequests + " pending requests at pri: " + highestPriRequest.getPriority() + " by preempting from " + numEntriesAtPreemptedPriority + " running tasks at priority: " + preemptedTaskPriority); // found something to preempt. get others of the same priority preemptedContainers = new ContainerId[numPendingRequestsToService]; int currIndex = 0; for (Map.Entry<Object, Container> entry : taskAllocations.entrySet()) { HeldContainer heldContainer = heldContainers.get(entry.getValue().getId()); CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo(); Priority taskPriority = lastTaskInfo.getPriority(); Container container = entry.getValue(); if (preemptedTaskPriority.equals(taskPriority)) { // taskAllocations map will iterate from oldest to newest assigned containers // keep the N newest containersIds with the matching priority preemptedContainers[currIndex++ % numPendingRequestsToService] = container.getId(); } } // app client will be notified when after container is killed // and we get its completed container status } } // upcall outside locks if (preemptedContainers != null) { heartbeatAtLastPreemption = numHeartbeats; for (int i = 0; i < numPendingRequestsToService; ++i) { ContainerId cId = preemptedContainers[i]; if (cId != null) { LOG.info("Preempting container: " + cId + " currently allocated to a task."); appClientDelegate.preemptContainer(cId); } } } }
From source file:org.apache.tez.mapreduce.hadoop.TestMRHelpers.java
License:Apache License
@Test(timeout = 5000) public void testContainerResourceConstruction() { JobConf conf = new JobConf(new Configuration()); Resource mapResource = MRHelpers.getResourceForMRMapper(conf); Resource reduceResource = MRHelpers.getResourceForMRReducer(conf); Assert.assertEquals(MRJobConfig.DEFAULT_MAP_CPU_VCORES, mapResource.getVirtualCores()); Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB, mapResource.getMemory()); Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_CPU_VCORES, reduceResource.getVirtualCores()); Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, reduceResource.getMemory()); conf.setInt(MRJobConfig.MAP_CPU_VCORES, 2); conf.setInt(MRJobConfig.MAP_MEMORY_MB, 123); conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 20); conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 1234); mapResource = MRHelpers.getResourceForMRMapper(conf); reduceResource = MRHelpers.getResourceForMRReducer(conf); Assert.assertEquals(2, mapResource.getVirtualCores()); Assert.assertEquals(123, mapResource.getMemory()); Assert.assertEquals(20, reduceResource.getVirtualCores()); Assert.assertEquals(1234, reduceResource.getMemory()); }
From source file:org.apache.twill.internal.yarn.Hadoop20YarnAMClient.java
License:Apache License
@Override protected Resource adjustCapability(Resource resource) { int cores = YarnUtils.getVirtualCores(resource); int updatedCores = Math.max(Math.min(cores, YarnUtils.getVirtualCores(maxCapability)), YarnUtils.getVirtualCores(minCapability)); // Try and set the virtual cores, which older versions of YARN don't support this. if (cores != updatedCores && YarnUtils.setVirtualCores(resource, updatedCores)) { LOG.info("Adjust virtual cores requirement from {} to {}.", cores, updatedCores); }/*w w w .j a v a 2 s . com*/ int updatedMemory = Math.min(resource.getMemory(), maxCapability.getMemory()); int minMemory = minCapability.getMemory(); updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory; if (resource.getMemory() != updatedMemory) { resource.setMemory(updatedMemory); LOG.info("Adjust memory requirement from {} to {} MB.", resource.getMemory(), updatedMemory); } return resource; }
From source file:org.apache.twill.internal.yarn.Hadoop20YarnAppClient.java
License:Apache License
@Override public ProcessLauncher<ApplicationMasterInfo> createLauncher(TwillSpecification twillSpec, @Nullable String schedulerQueue) throws Exception { // Request for new application final GetNewApplicationResponse response = yarnClient.getNewApplication(); final ApplicationId appId = response.getApplicationId(); // Setup the context for application submission final ApplicationSubmissionContext appSubmissionContext = Records .newRecord(ApplicationSubmissionContext.class); appSubmissionContext.setApplicationId(appId); appSubmissionContext.setApplicationName(twillSpec.getName()); appSubmissionContext.setUser(user);//from ww w . java 2 s .com if (schedulerQueue != null) { appSubmissionContext.setQueue(schedulerQueue); } // TODO: Make it adjustable through TwillSpec (TWILL-90) // Set the resource requirement for AM Resource amResource = Records.newRecord(Resource.class); amResource.setMemory(Constants.APP_MASTER_MEMORY_MB); final Resource capability = adjustMemory(response, amResource); ApplicationMasterInfo appMasterInfo = new ApplicationMasterInfo(appId, capability.getMemory(), 1); ApplicationSubmitter submitter = new ApplicationSubmitter() { @Override public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext) { ContainerLaunchContext context = launchContext.getLaunchContext(); addRMToken(context); context.setUser(appSubmissionContext.getUser()); context.setResource(adjustMemory(response, capability)); appSubmissionContext.setAMContainerSpec(context); try { yarnClient.submitApplication(appSubmissionContext); return new ProcessControllerImpl(yarnClient, appId); } catch (YarnRemoteException e) { LOG.error("Failed to submit application {}", appId, e); throw Throwables.propagate(e); } } }; return new ApplicationMasterProcessLauncher(appMasterInfo, submitter); }