Example usage for org.apache.hadoop.yarn.api.records Resource getVirtualCores

List of usage examples for org.apache.hadoop.yarn.api.records Resource getVirtualCores

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. Resource
  6. getVirtualCores

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Resource getVirtualCores.

Prototype

@Public
@Evolving
public abstract int getVirtualCores();

Source Link

Document

Get number of virtual cpu cores of the resource.

Usage

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughVCoreForApplicationMaster() throws Exception {
    int appMasterMem = 1024;
    int appMasterCore = 2;
    Resource resource = Resource.newInstance(appMasterMem, appMasterCore / 2);
    generateMaxAllocation(resource);/*from   ww w. j  a  va 2 s  . c om*/
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("ApplicationMaster",
            "virtual cores", appMasterCore, resource.getVirtualCores()));
    String[] args = new String[] { "-resource_path", "test", "-am_memory", Integer.toString(appMasterMem),
            "-am_vcores", Integer.toString(appMasterCore) };
    Client client = new Client(args);
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughVCoreForAlluxioMaster() throws Exception {
    Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM, "2048.00MB");
    Configuration.set(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU, "4");
    int masterMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM)
            / Constants.MB);//from www.  j a  v  a  2 s .  c  o  m
    int masterVCores = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU);
    Resource resource = Resource.newInstance(masterMemInMB, 3);
    generateMaxAllocation(resource);
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Master",
            "virtual cores", masterVCores, resource.getVirtualCores()));
    Client client = new Client();
    client.run();
}

From source file:alluxio.yarn.ClientTest.java

License:Apache License

@Test
public void notEnoughVCoreForAlluxioWorker() throws Exception {
    Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM, "2048.00MB");
    Configuration.set(PropertyKey.WORKER_MEMORY_SIZE, "4096.00MB");
    Configuration.set(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU, "8");
    int workerMemInMB = (int) (Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM)
            / Constants.MB);/*w w  w .ja va2s .c o m*/
    int ramdiskMemInMB = (int) (Configuration.getBytes(PropertyKey.WORKER_MEMORY_SIZE) / Constants.MB);
    int workerVCore = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU);
    Resource resource = Resource.newInstance((workerMemInMB + ramdiskMemInMB), 4);
    generateMaxAllocation(resource);
    mThrown.expect(RuntimeException.class);
    mThrown.expectMessage(ExceptionMessage.YARN_NOT_ENOUGH_RESOURCES.getMessage("Alluxio Worker",
            "virtual cores", workerVCore, resource.getVirtualCores()));
    Client client = new Client();
    client.run();
}

From source file:co.cask.cdap.master.startup.YarnCheck.java

License:Apache License

private void checkResources(List<NodeReport> nodeReports) {
    LOG.info("Checking that YARN has enough resources to run all system services.");
    int memoryCapacity = 0;
    int vcoresCapacity = 0;
    int memoryUsed = 0;
    int vcoresUsed = 0;
    int availableNodes = 0;
    for (NodeReport nodeReport : nodeReports) {
        NodeId nodeId = nodeReport.getNodeId();
        LOG.debug("Got report for node {}", nodeId);
        if (!nodeReport.getNodeState().isUnusable()) {
            Resource nodeCapability = nodeReport.getCapability();
            Resource nodeUsed = nodeReport.getUsed();

            // some versions of hadoop return null, others do not
            if (nodeCapability != null) {
                LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId,
                        nodeCapability.getMemory(), nodeCapability.getVirtualCores());
                memoryCapacity += nodeCapability.getMemory();
                vcoresCapacity += nodeCapability.getVirtualCores();
            }/*w w w  . j a  v a 2  s  . co m*/

            if (nodeUsed != null) {
                LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(),
                        nodeUsed.getVirtualCores());
                memoryUsed += nodeUsed.getMemory();
                vcoresUsed += nodeUsed.getVirtualCores();
            }

            availableNodes++;
        }
    }
    LOG.debug("YARN resource capacity: {} MB of memory and {} virtual cores.", memoryCapacity, vcoresCapacity);
    LOG.debug("YARN resources used: {} MB of memory and {} virtual cores.", memoryUsed, vcoresUsed);

    // calculate memory and vcores required by CDAP
    int requiredMemoryMB = 0;
    int requiredVCores = 0;
    Set<String> invalidKeys = new HashSet<>();
    for (ServiceResourceKeys serviceResourceKeys : systemServicesResourceKeys) {
        boolean hasConfigError = false;
        int instances = 0;
        int memoryMB = 0;
        int vcores = 0;

        try {
            instances = cConf.getInt(serviceResourceKeys.getInstancesKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getInstancesKey());
            hasConfigError = true;
        }
        try {
            memoryMB = cConf.getInt(serviceResourceKeys.getMemoryKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getMemoryKey());
            hasConfigError = true;
        }
        try {
            vcores = cConf.getInt(serviceResourceKeys.getVcoresKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getVcoresKey());
            hasConfigError = true;
        }

        if (!hasConfigError) {
            LOG.debug("Resource settings for system service {}: {}={}, {}={}, {}={}",
                    serviceResourceKeys.getServiceName(), serviceResourceKeys.getInstancesKey(), instances,
                    serviceResourceKeys.getMemoryKey(), memoryMB, serviceResourceKeys.getVcoresKey(), vcores);
            requiredMemoryMB += memoryMB * instances;
            requiredVCores += vcores * instances;
        }
    }

    if (!invalidKeys.isEmpty()) {
        throw new RuntimeException("YARN resources check failed to invalid config settings for keys: "
                + Joiner.on(',').join(invalidKeys));
    }

    LOG.debug("{} MB of memory and {} virtual cores are required.", requiredMemoryMB, requiredVCores);
    int availableMemoryMB = memoryCapacity - memoryUsed;
    int availableVCores = vcoresCapacity - vcoresUsed;
    boolean memoryOK = requiredMemoryMB <= availableMemoryMB;
    // if this is negative or zero just assume its not using vcores
    boolean vcoresOK = vcoresCapacity <= 0 || requiredVCores <= availableVCores;

    if (!memoryOK && !vcoresOK) {
        LOG.warn(
                "Services require {} MB of memory and {} vcores, "
                        + "but the cluster only has {} MB of memory and {} vcores available.",
                requiredMemoryMB, requiredVCores, availableMemoryMB, availableVCores);
    } else if (!memoryOK) {
        LOG.warn("Services require {} MB of memory but the cluster only has {} MB of memory available.",
                requiredMemoryMB, availableMemoryMB);
    } else if (!vcoresOK) {
        LOG.warn("Services require {} vcores but the cluster only has {} vcores available.", requiredVCores,
                availableVCores);
    } else {
        LOG.info("  YARN resources successfully verified.");
    }
}

From source file:co.cask.cdap.operations.yarn.YarnResources.java

License:Apache License

@Override
public synchronized void collect() throws Exception {
    reset();//  w w  w . j  a  v  a  2 s  .c  om
    List<NodeReport> nodeReports;
    YarnClient yarnClient = createYARNClient();
    try {
        nodeReports = yarnClient.getNodeReports();
    } finally {
        yarnClient.stop();
    }
    for (NodeReport nodeReport : nodeReports) {
        NodeId nodeId = nodeReport.getNodeId();
        LOG.debug("Got report for node {}", nodeId);
        if (!nodeReport.getNodeState().isUnusable()) {
            Resource nodeCapability = nodeReport.getCapability();
            Resource nodeUsed = nodeReport.getUsed();

            // some versions of hadoop return null, others do not
            if (nodeCapability != null) {
                LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId,
                        nodeCapability.getMemory(), nodeCapability.getVirtualCores());
                totalMemory += nodeCapability.getMemory();
                totalVCores += nodeCapability.getVirtualCores();
            }

            if (nodeUsed != null) {
                LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(),
                        nodeUsed.getVirtualCores());
                usedMemory += nodeUsed.getMemory();
                usedVCores += nodeUsed.getVirtualCores();
            }
        }
    }
}

From source file:com.cloudera.kitten.lua.AsapLuaContainerLaunchParameters.java

License:Open Source License

@Override
public Resource getContainerResource(Resource clusterMax) {
    LOG.info("Max cores: " + clusterMax.getVirtualCores());
    LOG.info("Max memory: " + clusterMax.getMemory());
    Resource rsrc = Records.newRecord(Resource.class);
    rsrc.setMemory(Math.min(clusterMax.getMemory(), getMemory()));
    LOG.info("Setting cores: " + getCores());
    rsrc.setVirtualCores(Math.min(clusterMax.getVirtualCores(), getCores()));
    return rsrc;//from  w ww.  j av  a 2s .co  m
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public List<NodeInfo> getNodes() throws LlamaException {
    List<NodeInfo> ret = new ArrayList<NodeInfo>();
    try {/*from ww w  . j av  a2s. co  m*/
        if (nodes == null) {
            // Get it from the yarn client.
            List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
            for (NodeReport nodeReport : nodeReports) {
                Resource resource = nodeReport.getCapability();
                NodeInfo nodeInfo = new NodeInfo(getNodeName(nodeReport.getNodeId()),
                        resource.getVirtualCores(), resource.getMemory());
                ret.add(nodeInfo);
            }
        } else {
            // Get it from the nodes structure which is being kept upto date.
            synchronized (nodes) {
                for (Map.Entry<String, Resource> entry : nodes.entrySet()) {
                    Resource nodeReport = entry.getValue();
                    NodeInfo nodeInfo = new NodeInfo(entry.getKey(), nodeReport.getVirtualCores(),
                            nodeReport.getMemory());
                    ret.add(nodeInfo);
                }
            }
        }
        return ret;
    } catch (Throwable ex) {
        throw new LlamaException(ex, ErrorCode.AM_CANNOT_GET_NODES, appId);
    }
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void verifyResources(Collection<RMResource> resources) throws LlamaException {
    for (RMResource r : resources) {
        Resource nodeCapabilites = nodes.get(r.getLocationAsk());
        if (nodeCapabilites == null) {
            throw new LlamaException(ErrorCode.AM_NODE_NOT_AVAILABLE, appId, r.getLocationAsk(), r);
        }/* www.  j  av a  2  s  .  c o m*/
        if (r.getCpuVCoresAsk() > maxResource.getVirtualCores()) {
            throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_MAX_CPUS, appId, r,
                    maxResource.getVirtualCores());
        }
        if (r.getMemoryMbsAsk() > maxResource.getMemory()) {
            throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_MAX_MEMORY, appId, r, maxResource.getMemory());
        }
        if (r.getCpuVCoresAsk() > nodeCapabilites.getVirtualCores()) {
            throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_NODE_CPUS, appId, r,
                    nodeCapabilites.getVirtualCores());
        }
        if (r.getMemoryMbsAsk() > nodeCapabilites.getMemory()) {
            throw new LlamaException(ErrorCode.AM_RESOURCE_OVER_NODE_MEMORY, appId, r,
                    nodeCapabilites.getMemory());
        }
    }
}

From source file:com.continuuity.weave.internal.yarn.Hadoop21YarnAMClient.java

License:Apache License

private Resource adjustCapability(Resource resource) {
    int cores = resource.getVirtualCores();
    int updatedCores = Math.min(resource.getVirtualCores(), maxCapability.getVirtualCores());

    if (cores != updatedCores) {
        resource.setVirtualCores(updatedCores);
        LOG.info("Adjust virtual cores requirement from {} to {}.", cores, updatedCores);
    }//w w  w.j  av  a  2s  . c o  m

    int updatedMemory = Math.min(resource.getMemory(), maxCapability.getMemory());
    if (resource.getMemory() != updatedMemory) {
        resource.setMemory(updatedMemory);
        LOG.info("Adjust memory requirement from {} to {} MB.", resource.getMemory(), updatedMemory);
    }

    return resource;
}

From source file:com.hazelcast.yarn.ApplicationMaster.java

License:Open Source License

private boolean checkResources() {
    Resource availableRes = this.rmClient.getAvailableResources();

    return availableRes == null || availableRes.getMemory() >= this.properties.memoryPerNode()
            && availableRes.getVirtualCores() >= this.properties.cpuPerNode();
}