List of usage examples for org.apache.hadoop.yarn.api.records Resource getMemory
@Public @Deprecated public abstract int getMemory();
From source file:alluxio.yarn.ClientTest.java
License:Apache License
@Test public void notEnoughMemoryForApplicationMaster() throws Exception { int appMasterMem = 1024; Resource resource = Resource.newInstance(appMasterMem / 2, 4); generateMaxAllocation(resource);//from w ww .ja va 2 s . co m mThrown.expect(RuntimeException.class); mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("ApplicationMaster", "memory", appMasterMem, resource.getMemory())); String[] args = new String[] { "-resource_path", "test", "-am_memory", Integer.toString(appMasterMem), "-am_vcores", "2" }; Client client = new Client(args); client.run(); }
From source file:alluxio.yarn.ClientTest.java
License:Apache License
@Test public void notEnoughMemoryForAlluxioMaster() throws Exception { Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM, "2048.00MB"); Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU, "4"); int masterMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM) / Constants.MB);/* w ww .jav a 2 s .c o m*/ Resource resource = Resource.newInstance(masterMemInMB / 2, 4); generateMaxAllocation(resource); mThrown.expect(RuntimeException.class); mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Master", "memory", masterMemInMB, resource.getMemory())); Client client = new Client(); client.run(); }
From source file:alluxio.yarn.ClientTest.java
License:Apache License
@Test public void notEnoughMemoryForAlluxioWorker() throws Exception { Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM, "2048.00MB"); Configuration.set(PropertyKey.WORKER_MEMORY_SIZE, "4096.00MB"); Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU, "8"); int workerMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM) / Constants.MB);/*from w w w. ja v a2s .c o m*/ int ramdiskMemInMB = (int) (Configuration.getBytes(PropertyKey.WORKER_MEMORY_SIZE) / Constants.MB); Resource resource = Resource.newInstance((workerMemInMB + ramdiskMemInMB) / 2, 4); generateMaxAllocation(resource); mThrown.expect(RuntimeException.class); mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Worker", "memory", (workerMemInMB + ramdiskMemInMB), resource.getMemory())); Client client = new Client(); client.run(); }
From source file:co.cask.cdap.master.startup.YarnCheck.java
License:Apache License
private void checkResources(List<NodeReport> nodeReports) { LOG.info("Checking that YARN has enough resources to run all system services."); int memoryCapacity = 0; int vcoresCapacity = 0; int memoryUsed = 0; int vcoresUsed = 0; int availableNodes = 0; for (NodeReport nodeReport : nodeReports) { NodeId nodeId = nodeReport.getNodeId(); LOG.debug("Got report for node {}", nodeId); if (!nodeReport.getNodeState().isUnusable()) { Resource nodeCapability = nodeReport.getCapability(); Resource nodeUsed = nodeReport.getUsed(); // some versions of hadoop return null, others do not if (nodeCapability != null) { LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId, nodeCapability.getMemory(), nodeCapability.getVirtualCores()); memoryCapacity += nodeCapability.getMemory(); vcoresCapacity += nodeCapability.getVirtualCores(); }/*from ww w . j a v a2s . com*/ if (nodeUsed != null) { LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(), nodeUsed.getVirtualCores()); memoryUsed += nodeUsed.getMemory(); vcoresUsed += nodeUsed.getVirtualCores(); } availableNodes++; } } LOG.debug("YARN resource capacity: {} MB of memory and {} virtual cores.", memoryCapacity, vcoresCapacity); LOG.debug("YARN resources used: {} MB of memory and {} virtual cores.", memoryUsed, vcoresUsed); // calculate memory and vcores required by CDAP int requiredMemoryMB = 0; int requiredVCores = 0; Set<String> invalidKeys = new HashSet<>(); for (ServiceResourceKeys serviceResourceKeys : systemServicesResourceKeys) { boolean hasConfigError = false; int instances = 0; int memoryMB = 0; int vcores = 0; try { instances = cConf.getInt(serviceResourceKeys.getInstancesKey()); } catch (Exception e) { invalidKeys.add(serviceResourceKeys.getInstancesKey()); hasConfigError = true; } try { memoryMB = cConf.getInt(serviceResourceKeys.getMemoryKey()); } catch (Exception e) { invalidKeys.add(serviceResourceKeys.getMemoryKey()); hasConfigError = true; } try { vcores = cConf.getInt(serviceResourceKeys.getVcoresKey()); } catch (Exception e) { invalidKeys.add(serviceResourceKeys.getVcoresKey()); hasConfigError = true; } if (!hasConfigError) { LOG.debug("Resource settings for system service {}: {}={}, {}={}, {}={}", serviceResourceKeys.getServiceName(), serviceResourceKeys.getInstancesKey(), instances, serviceResourceKeys.getMemoryKey(), memoryMB, serviceResourceKeys.getVcoresKey(), vcores); requiredMemoryMB += memoryMB * instances; requiredVCores += vcores * instances; } } if (!invalidKeys.isEmpty()) { throw new RuntimeException("YARN resources check failed to invalid config settings for keys: " + Joiner.on(',').join(invalidKeys)); } LOG.debug("{} MB of memory and {} virtual cores are required.", requiredMemoryMB, requiredVCores); int availableMemoryMB = memoryCapacity - memoryUsed; int availableVCores = vcoresCapacity - vcoresUsed; boolean memoryOK = requiredMemoryMB <= availableMemoryMB; // if this is negative or zero just assume its not using vcores boolean vcoresOK = vcoresCapacity <= 0 || requiredVCores <= availableVCores; if (!memoryOK && !vcoresOK) { LOG.warn( "Services require {} MB of memory and {} vcores, " + "but the cluster only has {} MB of memory and {} vcores available.", requiredMemoryMB, requiredVCores, availableMemoryMB, availableVCores); } else if (!memoryOK) { LOG.warn("Services require {} MB of memory but the cluster only has {} MB of memory available.", requiredMemoryMB, availableMemoryMB); } else if (!vcoresOK) { LOG.warn("Services require {} vcores but the cluster only has {} vcores available.", requiredVCores, availableVCores); } else { LOG.info(" YARN resources successfully verified."); } }
From source file:co.cask.cdap.operations.yarn.YarnResources.java
License:Apache License
@Override public synchronized void collect() throws Exception { reset();//from www. ja v a2s. co m List<NodeReport> nodeReports; YarnClient yarnClient = createYARNClient(); try { nodeReports = yarnClient.getNodeReports(); } finally { yarnClient.stop(); } for (NodeReport nodeReport : nodeReports) { NodeId nodeId = nodeReport.getNodeId(); LOG.debug("Got report for node {}", nodeId); if (!nodeReport.getNodeState().isUnusable()) { Resource nodeCapability = nodeReport.getCapability(); Resource nodeUsed = nodeReport.getUsed(); // some versions of hadoop return null, others do not if (nodeCapability != null) { LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId, nodeCapability.getMemory(), nodeCapability.getVirtualCores()); totalMemory += nodeCapability.getMemory(); totalVCores += nodeCapability.getVirtualCores(); } if (nodeUsed != null) { LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(), nodeUsed.getVirtualCores()); usedMemory += nodeUsed.getMemory(); usedVCores += nodeUsed.getVirtualCores(); } } } }
From source file:com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java
License:Open Source License
@Override public Resource getContainerResource(Resource clusterMax) { LOG.info("Max cores: " + clusterMax.getVirtualCores()); LOG.info("Max memory: " + clusterMax.getMemory()); Resource rsrc = Records.newRecord(Resource.class); rsrc.setMemory(Math.min(clusterMax.getMemory(), getMemory())); LOG.info("Setting cores: " + getCores()); rsrc.setVirtualCores(Math.min(clusterMax.getVirtualCores(), getCores())); return rsrc;//w w w . jav a 2 s. co m }
From source file:com.cloudera.kitten.lua.LuaContainerLaunchParameters.java
License:Open Source License
@Override public Resource getContainerResource(Resource clusterMin, Resource clusterMax) { Resource rsrc = Records.newRecord(Resource.class); rsrc.setMemory(Math.min(clusterMax.getMemory(), Math.max(clusterMin.getMemory(), getMemory()))); return rsrc;//from ww w .ja v a 2 s. c om }
From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java
License:Apache License
@Override public List<NodeInfo> getNodes() throws LlamaException { List<NodeInfo> ret = new ArrayList<NodeInfo>(); try {//w w w . j av a 2 s . co m if (nodes == null) { // Get it from the yarn client. List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport nodeReport : nodeReports) { Resource resource = nodeReport.getCapability(); NodeInfo nodeInfo = new NodeInfo(getNodeName(nodeReport.getNodeId()), resource.getVirtualCores(), resource.getMemory()); ret.add(nodeInfo); } } else { // Get it from the nodes structure which is being kept upto date. synchronized (nodes) { for (Map.Entry<String, Resource> entry : nodes.entrySet()) { Resource nodeReport = entry.getValue(); NodeInfo nodeInfo = new NodeInfo(entry.getKey(), nodeReport.getVirtualCores(), nodeReport.getMemory()); ret.add(nodeInfo); } } } return ret; } catch (Throwable ex) { throw new LlamaException(ex, ErrorCode.AM_CANNOT_GET_NODES, appId); } }
From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java
License:Apache License
private void verifyResources(Collection<RMResource> resources) throws LlamaException { for (RMResource r : resources) { Resource nodeCapabilites = nodes.get(r.getLocationAsk()); if (nodeCapabilites == null) { throw new LlamaException(ErrorCode.AM_NODE_NOT_AVAILABLE, appId, r.getLocationAsk(), r); }/*from www.j a v a 2s . c om*/ if (r.getCpuVCoresAsk() > maxResource.getVirtualCores()) { throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_MAX_CPUS, appId, r, maxResource.getVirtualCores()); } if (r.getMemoryMbsAsk() > maxResource.getMemory()) { throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_MAX_MEMORY, appId, r, maxResource.getMemory()); } if (r.getCpuVCoresAsk() > nodeCapabilites.getVirtualCores()) { throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_NODE_CPUS, appId, r, nodeCapabilites.getVirtualCores()); } if (r.getMemoryMbsAsk() > nodeCapabilites.getMemory()) { throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_NODE_MEMORY, appId, r, nodeCapabilites.getMemory()); } } }
From source file:com.continuuity.weave.internal.yarn.Hadoop20YarnAMClient.java
License:Apache License
private Resource adjustCapability(Resource resource) { int cores = YarnUtils.getVirtualCores(resource); int updatedCores = Math.max(Math.min(cores, YarnUtils.getVirtualCores(maxCapability)), YarnUtils.getVirtualCores(minCapability)); // Try and set the virtual cores, which older versions of YARN don't support this. if (cores != updatedCores && YarnUtils.setVirtualCores(resource, updatedCores)) { LOG.info("Adjust virtual cores requirement from {} to {}.", cores, updatedCores); }//from w w w . j a va 2 s . c om int updatedMemory = Math.min(resource.getMemory(), maxCapability.getMemory()); int minMemory = minCapability.getMemory(); updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory; if (resource.getMemory() != updatedMemory) { resource.setMemory(updatedMemory); LOG.info("Adjust memory requirement from {} to {} MB.", resource.getMemory(), updatedMemory); } return resource; }