Example usage for org.apache.hadoop.yarn.api.records NodeReport getCapability

List of usage examples for org.apache.hadoop.yarn.api.records NodeReport getCapability

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. NodeReport
  6. getCapability

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records NodeReport getCapability.

Prototype

@Public
@Stable
public abstract Resource getCapability();

Source Link

Document

Get the total Resource on the node.

Usage

From source file:cn.edu.buaa.act.petuumOnYarn.ApplicationMaster.java

License:Apache License

private boolean getAvaliableNodes() {
    List<NodeReport> clusterNodeReports;
    try {//w w  w.  j a va 2  s .co  m
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(conf);
        yarnClient.start();

        clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
        for (NodeReport node : clusterNodeReports) {
            LOG.info("node infos:" + node.getHttpAddress());
        }

        avaliableNodeList = new ArrayList<NodeReport>();
        if (numNodes <= clusterNodeReports.size()) {
            for (NodeReport node : clusterNodeReports) {
                if (node.getCapability().getMemory() >= containerMemory
                        && node.getCapability().getVirtualCores() >= containerVirtualCores) {
                    avaliableNodeList.add(node);
                }
            }
            if (avaliableNodeList.size() >= numNodes)
                numTotalContainers = numNodes;
            else {
                LOG.error("Resource isn't enough");
                return false;
            }
        } else {
            LOG.error("cluster nodes isn't enough");
            return false;
        }
    } catch (Exception e) {
        LOG.error(e.getMessage());
        LOG.error(e.getStackTrace());
        return false;
    }
    return true;
}

From source file:co.cask.cdap.master.startup.YarnCheck.java

License:Apache License

private void checkResources(List<NodeReport> nodeReports) {
    LOG.info("Checking that YARN has enough resources to run all system services.");
    int memoryCapacity = 0;
    int vcoresCapacity = 0;
    int memoryUsed = 0;
    int vcoresUsed = 0;
    int availableNodes = 0;
    for (NodeReport nodeReport : nodeReports) {
        NodeId nodeId = nodeReport.getNodeId();
        LOG.debug("Got report for node {}", nodeId);
        if (!nodeReport.getNodeState().isUnusable()) {
            Resource nodeCapability = nodeReport.getCapability();
            Resource nodeUsed = nodeReport.getUsed();

            // some versions of hadoop return null, others do not
            if (nodeCapability != null) {
                LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId,
                        nodeCapability.getMemory(), nodeCapability.getVirtualCores());
                memoryCapacity += nodeCapability.getMemory();
                vcoresCapacity += nodeCapability.getVirtualCores();
            }//www  . j  a va2s  . co  m

            if (nodeUsed != null) {
                LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(),
                        nodeUsed.getVirtualCores());
                memoryUsed += nodeUsed.getMemory();
                vcoresUsed += nodeUsed.getVirtualCores();
            }

            availableNodes++;
        }
    }
    LOG.debug("YARN resource capacity: {} MB of memory and {} virtual cores.", memoryCapacity, vcoresCapacity);
    LOG.debug("YARN resources used: {} MB of memory and {} virtual cores.", memoryUsed, vcoresUsed);

    // calculate memory and vcores required by CDAP
    int requiredMemoryMB = 0;
    int requiredVCores = 0;
    Set<String> invalidKeys = new HashSet<>();
    for (ServiceResourceKeys serviceResourceKeys : systemServicesResourceKeys) {
        boolean hasConfigError = false;
        int instances = 0;
        int memoryMB = 0;
        int vcores = 0;

        try {
            instances = cConf.getInt(serviceResourceKeys.getInstancesKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getInstancesKey());
            hasConfigError = true;
        }
        try {
            memoryMB = cConf.getInt(serviceResourceKeys.getMemoryKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getMemoryKey());
            hasConfigError = true;
        }
        try {
            vcores = cConf.getInt(serviceResourceKeys.getVcoresKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getVcoresKey());
            hasConfigError = true;
        }

        if (!hasConfigError) {
            LOG.debug("Resource settings for system service {}: {}={}, {}={}, {}={}",
                    serviceResourceKeys.getServiceName(), serviceResourceKeys.getInstancesKey(), instances,
                    serviceResourceKeys.getMemoryKey(), memoryMB, serviceResourceKeys.getVcoresKey(), vcores);
            requiredMemoryMB += memoryMB * instances;
            requiredVCores += vcores * instances;
        }
    }

    if (!invalidKeys.isEmpty()) {
        throw new RuntimeException("YARN resources check failed to invalid config settings for keys: "
                + Joiner.on(',').join(invalidKeys));
    }

    LOG.debug("{} MB of memory and {} virtual cores are required.", requiredMemoryMB, requiredVCores);
    int availableMemoryMB = memoryCapacity - memoryUsed;
    int availableVCores = vcoresCapacity - vcoresUsed;
    boolean memoryOK = requiredMemoryMB <= availableMemoryMB;
    // if this is negative or zero just assume its not using vcores
    boolean vcoresOK = vcoresCapacity <= 0 || requiredVCores <= availableVCores;

    if (!memoryOK && !vcoresOK) {
        LOG.warn(
                "Services require {} MB of memory and {} vcores, "
                        + "but the cluster only has {} MB of memory and {} vcores available.",
                requiredMemoryMB, requiredVCores, availableMemoryMB, availableVCores);
    } else if (!memoryOK) {
        LOG.warn("Services require {} MB of memory but the cluster only has {} MB of memory available.",
                requiredMemoryMB, availableMemoryMB);
    } else if (!vcoresOK) {
        LOG.warn("Services require {} vcores but the cluster only has {} vcores available.", requiredVCores,
                availableVCores);
    } else {
        LOG.info("  YARN resources successfully verified.");
    }
}

From source file:co.cask.cdap.operations.yarn.YarnResources.java

License:Apache License

@Override
public synchronized void collect() throws Exception {
    reset();//from w w  w .j a  v  a2  s  .  c  om
    List<NodeReport> nodeReports;
    YarnClient yarnClient = createYARNClient();
    try {
        nodeReports = yarnClient.getNodeReports();
    } finally {
        yarnClient.stop();
    }
    for (NodeReport nodeReport : nodeReports) {
        NodeId nodeId = nodeReport.getNodeId();
        LOG.debug("Got report for node {}", nodeId);
        if (!nodeReport.getNodeState().isUnusable()) {
            Resource nodeCapability = nodeReport.getCapability();
            Resource nodeUsed = nodeReport.getUsed();

            // some versions of hadoop return null, others do not
            if (nodeCapability != null) {
                LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId,
                        nodeCapability.getMemory(), nodeCapability.getVirtualCores());
                totalMemory += nodeCapability.getMemory();
                totalVCores += nodeCapability.getVirtualCores();
            }

            if (nodeUsed != null) {
                LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(),
                        nodeUsed.getVirtualCores());
                usedMemory += nodeUsed.getMemory();
                usedVCores += nodeUsed.getVirtualCores();
            }
        }
    }
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void _registerSchedulerAndCreateNMClient(String queue) throws Exception {
    NMTokenCache nmTokenCache = new NMTokenCache();
    nmClient = NMClient.createNMClient();
    nmClient.setNMTokenCache(nmTokenCache);
    nmClient.init(yarnConf);/*www  .  j  a  v  a  2s .c o  m*/
    nmClient.start();
    LOG.debug("Started NMClient, AM '{}' with scheduler for '{}' queue", appId, queue);
    int heartbeatInterval = getConf().getInt(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERNAL_DEFAULT);
    AMRMClient<LlamaContainerRequest> amRmClient = AMRMClient.createAMRMClient();
    amRmClient.setNMTokenCache(nmTokenCache);
    amRmClientAsync = AMRMClientAsync.createAMRMClientAsync(amRmClient, heartbeatInterval,
            YarnRMConnector.this);
    amRmClientAsync.init(yarnConf);
    amRmClientAsync.start();
    String urlWithoutScheme = getConf().get(ADVERTISED_TRACKING_URL_KEY, "http://")
            .substring("http://".length());
    RegisterApplicationMasterResponse response = amRmClientAsync.registerApplicationMaster(
            getConf().get(ADVERTISED_HOSTNAME_KEY, ""), getConf().getInt(ADVERTISED_PORT_KEY, 0),
            urlWithoutScheme);
    maxResource = response.getMaximumResourceCapability();
    nodes = Collections.synchronizedMap(new HashMap<String, Resource>());
    for (NodeReport nodeReport : yarnClient.getNodeReports()) {
        if (nodeReport.getNodeState() == NodeState.RUNNING) {
            String nodeKey = getNodeName(nodeReport.getNodeId());
            nodes.put(nodeKey, nodeReport.getCapability());
            LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey,
                    nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory());
        }
    }
    LOG.debug("Registered with scheduler, AM '{}' for '{}' queue", appId, queue);
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public List<NodeInfo> getNodes() throws LlamaException {
    List<NodeInfo> ret = new ArrayList<NodeInfo>();
    try {//  ww w  .  j  a  v a2  s .  c  om
        if (nodes == null) {
            // Get it from the yarn client.
            List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
            for (NodeReport nodeReport : nodeReports) {
                Resource resource = nodeReport.getCapability();
                NodeInfo nodeInfo = new NodeInfo(getNodeName(nodeReport.getNodeId()),
                        resource.getVirtualCores(), resource.getMemory());
                ret.add(nodeInfo);
            }
        } else {
            // Get it from the nodes structure which is being kept upto date.
            synchronized (nodes) {
                for (Map.Entry<String, Resource> entry : nodes.entrySet()) {
                    Resource nodeReport = entry.getValue();
                    NodeInfo nodeInfo = new NodeInfo(entry.getKey(), nodeReport.getVirtualCores(),
                            nodeReport.getMemory());
                    ret.add(nodeInfo);
                }
            }
        }
        return ret;
    } catch (Throwable ex) {
        throw new LlamaException(ex, ErrorCode.AM_CANNOT_GET_NODES, appId);
    }
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public void onNodesUpdated(List<NodeReport> nodeReports) {
    LOG.debug("Received nodes update for '{}' nodes", nodeReports.size());
    for (NodeReport nodeReport : nodeReports) {
        if (nodeReport.getNodeState() == NodeState.RUNNING) {
            String nodeKey = getNodeName(nodeReport.getNodeId());
            nodes.put(nodeKey, nodeReport.getCapability());
            LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey,
                    nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory());
        } else {/*from  w w  w  . j  av a  2s  . c om*/
            LOG.debug("Removed node '{}'", nodeReport.getNodeId());
            nodes.remove(getNodeName(nodeReport.getNodeId()));
        }
    }
}

From source file:com.datatorrent.stram.ResourceRequestHandler.java

License:Apache License

/**
 * Tracks update to available resources. Resource availability is used to make decisions about where to request new
 * containers.//w ww  .  ja va2  s  .c o m
 *
 * @param nodeReports
 */
public void updateNodeReports(List<NodeReport> nodeReports) {
    // LOG.debug("Got {} updated node reports.", nodeReports.size());
    for (NodeReport nr : nodeReports) {
        StringBuilder sb = new StringBuilder();
        sb.append("rackName=").append(nr.getRackName()).append(",nodeid=").append(nr.getNodeId())
                .append(",numContainers=").append(nr.getNumContainers()).append(",capability=")
                .append(nr.getCapability()).append("used=").append(nr.getUsed()).append("state=")
                .append(nr.getNodeState());
        LOG.info("Node report: " + sb);
        nodeReportMap.put(nr.getNodeId().getHost(), nr);
        nodeToRack.put(nr.getNodeId().getHost(), nr.getRackName());
    }
}

From source file:com.datatorrent.stram.ResourceRequestHandler.java

License:Apache License

public String getHost(ContainerStartRequest csr, boolean first) {
    String host = null;//from   w ww. j av a  2  s .  co  m
    PTContainer c = csr.container;
    if (first) {
        for (PTOperator oper : c.getOperators()) {
            HostOperatorSet grpObj = oper.getNodeLocalOperators();
            host = nodeLocalMapping.get(grpObj.getOperatorSet());
            if (host != null) {
                return host;
            }
            if (grpObj.getHost() != null) {
                host = grpObj.getHost();
                // using the 1st host value as host for container
                break;
            }
        }
        if (host != null && nodeReportMap.get(host) != null) {
            for (PTOperator oper : c.getOperators()) {
                HostOperatorSet grpObj = oper.getNodeLocalOperators();
                Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
                NodeReport report = nodeReportMap.get(host);
                int aggrMemory = c.getRequiredMemoryMB();
                int vCores = c.getRequiredVCores();
                Set<PTContainer> containers = Sets.newHashSet();
                containers.add(c);
                for (PTOperator nodeLocalOper : nodeLocalSet) {
                    if (!containers.contains(nodeLocalOper.getContainer())) {
                        aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                        vCores += nodeLocalOper.getContainer().getRequiredVCores();
                        containers.add(nodeLocalOper.getContainer());
                    }
                }
                int memAvailable = report.getCapability().getMemory() - report.getUsed().getMemory();
                int vCoresAvailable = report.getCapability().getVirtualCores()
                        - report.getUsed().getVirtualCores();
                if (memAvailable >= aggrMemory && vCoresAvailable >= vCores) {
                    nodeLocalMapping.put(nodeLocalSet, host);
                    return host;
                }
            }
        }
    }

    // the host requested didn't have the resources so looking for other hosts
    host = null;
    for (PTOperator oper : c.getOperators()) {
        HostOperatorSet grpObj = oper.getNodeLocalOperators();
        Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
        if (nodeLocalSet.size() > 1) {
            LOG.debug("Finding new host for {}", nodeLocalSet);
            int aggrMemory = c.getRequiredMemoryMB();
            int vCores = c.getRequiredVCores();
            Set<PTContainer> containers = Sets.newHashSet();
            containers.add(c);
            // aggregate memory required for all containers
            for (PTOperator nodeLocalOper : nodeLocalSet) {
                if (!containers.contains(nodeLocalOper.getContainer())) {
                    aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                    vCores += nodeLocalOper.getContainer().getRequiredVCores();
                    containers.add(nodeLocalOper.getContainer());
                }
            }
            for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) {
                int memAvailable = nodeEntry.getValue().getCapability().getMemory()
                        - nodeEntry.getValue().getUsed().getMemory();
                int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores()
                        - nodeEntry.getValue().getUsed().getVirtualCores();
                if (memAvailable >= aggrMemory && vCoresAvailable >= vCores) {
                    host = nodeEntry.getKey();
                    grpObj.setHost(host);
                    nodeLocalMapping.put(nodeLocalSet, host);
                    return host;
                }
            }
        }
    }
    return host;
}

From source file:com.datatorrent.stram.StramMiniClusterTest.java

License:Apache License

@Test
public void testSetupShutdown() throws Exception {
    GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
    ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
    GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
    List<NodeReport> nodeReports = response.getNodeReports();
    LOG.info("{}", nodeReports);

    for (NodeReport nr : nodeReports) {
        LOG.info("Node: {}", nr.getNodeId());
        LOG.info("Total memory: {}", nr.getCapability());
        LOG.info("Used memory: {}", nr.getUsed());
        LOG.info("Number containers: {}", nr.getNumContainers());
    }//from  w  w  w . jav a2s. c  o  m

    String appMasterJar = JarFinder.getJar(StreamingAppMaster.class);
    LOG.info("appmaster jar: " + appMasterJar);
    String testJar = JarFinder.getJar(StramMiniClusterTest.class);
    LOG.info("testJar: " + testJar);

    // create test application
    Properties dagProps = new Properties();

    // input module (ensure shutdown works while windows are generated)
    dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.classname",
            TestGeneratorInputOperator.class.getName());
    dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.maxTuples", "1");

    // fake output adapter - to be ignored when determine shutdown
    //props.put(DAGContext.DT_PREFIX + "stream.output.classname", HDFSOutputStream.class.getName());
    //props.put(DAGContext.DT_PREFIX + "stream.output.inputNode", "module2");
    //props.put(DAGContext.DT_PREFIX + "stream.output.filepath", "miniclustertest-testSetupShutdown.out");

    dagProps.put(StreamingApplication.DT_PREFIX + "operator.module1.classname",
            GenericTestOperator.class.getName());

    dagProps.put(StreamingApplication.DT_PREFIX + "operator.module2.classname",
            GenericTestOperator.class.getName());

    dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.source", "numGen.outport");
    dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.sinks", "module1.inport1");

    dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.source", "module1.outport1");
    dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.sinks", "module2.inport1");

    dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(),
            "-Dlog4j.properties=custom_log4j.properties");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(),
            "64");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.VCORES.getName(),
            "1");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + "operator.*.port.*."
            + Context.PortContext.BUFFER_MEMORY_MB.getName(), "32");
    dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.DEBUG.getName(), "true");
    //dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINERS_MAX_COUNT.getName(), "2");
    LOG.info("dag properties: {}", dagProps);

    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf);
    tb.addFromProperties(dagProps, null);
    LogicalPlan dag = createDAG(tb);
    Configuration yarnConf = new Configuration(yarnCluster.getConfig());
    StramClient client = new StramClient(yarnConf, dag);
    try {
        client.start();
        if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
            client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
        }
        LOG.info("Running client");
        client.startApplication();
        boolean result = client.monitorApplication();

        LOG.info("Client run completed. Result=" + result);
        Assert.assertTrue(result);
    } finally {
        client.stop();
    }
}

From source file:com.google.mr4c.hadoop.yarn.YarnBinding.java

License:Open Source License

public static List<ResourceLimit> computeResourceLimits(Iterable<NodeReport> reports) {
    if (!reports.iterator().hasNext()) {
        return Collections.emptyList();
    }/*w w  w . j  av  a2 s .c  o  m*/

    List<ResourceLimit> limits = new ArrayList<ResourceLimit>();
    int cores = 0;
    int memory = 0;
    int gpu_memory = 0;

    for (NodeReport report : reports) {
        org.apache.hadoop.yarn.api.records.Resource cap = report.getCapability();
        cores = Math.max(cores, cap.getVirtualCores());
        memory = Math.max(memory, cap.getMemory());
        gpu_memory = Math.max(gpu_memory, cap.getGpuMemory());
    }
    limits.add(new ResourceLimit(Resource.CORES, cores, LimitSource.CLUSTER));
    limits.add(new ResourceLimit(Resource.MEMORY, memory, LimitSource.CLUSTER));
    limits.add(new ResourceLimit(Resource.GPU_MEMORY, gpu_memory, LimitSource.CLUSTER));
    return limits;
}