Example usage for org.apache.hadoop.yarn.api.records NodeReport getNodeState

List of usage examples for org.apache.hadoop.yarn.api.records NodeReport getNodeState

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. NodeReport
  6. getNodeState

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records NodeReport getNodeState.

Prototype

@Public
@Stable
public abstract NodeState getNodeState();

Source Link

Document

Get the NodeState of the node.

Usage

From source file:co.cask.cdap.master.startup.YarnCheck.java

License:Apache License

private void checkResources(List<NodeReport> nodeReports) {
    LOG.info("Checking that YARN has enough resources to run all system services.");
    int memoryCapacity = 0;
    int vcoresCapacity = 0;
    int memoryUsed = 0;
    int vcoresUsed = 0;
    int availableNodes = 0;
    for (NodeReport nodeReport : nodeReports) {
        NodeId nodeId = nodeReport.getNodeId();
        LOG.debug("Got report for node {}", nodeId);
        if (!nodeReport.getNodeState().isUnusable()) {
            Resource nodeCapability = nodeReport.getCapability();
            Resource nodeUsed = nodeReport.getUsed();

            // some versions of hadoop return null, others do not
            if (nodeCapability != null) {
                LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId,
                        nodeCapability.getMemory(), nodeCapability.getVirtualCores());
                memoryCapacity += nodeCapability.getMemory();
                vcoresCapacity += nodeCapability.getVirtualCores();
            }//w  w  w  .  ja  v  a2s  .c  om

            if (nodeUsed != null) {
                LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(),
                        nodeUsed.getVirtualCores());
                memoryUsed += nodeUsed.getMemory();
                vcoresUsed += nodeUsed.getVirtualCores();
            }

            availableNodes++;
        }
    }
    LOG.debug("YARN resource capacity: {} MB of memory and {} virtual cores.", memoryCapacity, vcoresCapacity);
    LOG.debug("YARN resources used: {} MB of memory and {} virtual cores.", memoryUsed, vcoresUsed);

    // calculate memory and vcores required by CDAP
    int requiredMemoryMB = 0;
    int requiredVCores = 0;
    Set<String> invalidKeys = new HashSet<>();
    for (ServiceResourceKeys serviceResourceKeys : systemServicesResourceKeys) {
        boolean hasConfigError = false;
        int instances = 0;
        int memoryMB = 0;
        int vcores = 0;

        try {
            instances = cConf.getInt(serviceResourceKeys.getInstancesKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getInstancesKey());
            hasConfigError = true;
        }
        try {
            memoryMB = cConf.getInt(serviceResourceKeys.getMemoryKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getMemoryKey());
            hasConfigError = true;
        }
        try {
            vcores = cConf.getInt(serviceResourceKeys.getVcoresKey());
        } catch (Exception e) {
            invalidKeys.add(serviceResourceKeys.getVcoresKey());
            hasConfigError = true;
        }

        if (!hasConfigError) {
            LOG.debug("Resource settings for system service {}: {}={}, {}={}, {}={}",
                    serviceResourceKeys.getServiceName(), serviceResourceKeys.getInstancesKey(), instances,
                    serviceResourceKeys.getMemoryKey(), memoryMB, serviceResourceKeys.getVcoresKey(), vcores);
            requiredMemoryMB += memoryMB * instances;
            requiredVCores += vcores * instances;
        }
    }

    if (!invalidKeys.isEmpty()) {
        throw new RuntimeException("YARN resources check failed to invalid config settings for keys: "
                + Joiner.on(',').join(invalidKeys));
    }

    LOG.debug("{} MB of memory and {} virtual cores are required.", requiredMemoryMB, requiredVCores);
    int availableMemoryMB = memoryCapacity - memoryUsed;
    int availableVCores = vcoresCapacity - vcoresUsed;
    boolean memoryOK = requiredMemoryMB <= availableMemoryMB;
    // if this is negative or zero just assume its not using vcores
    boolean vcoresOK = vcoresCapacity <= 0 || requiredVCores <= availableVCores;

    if (!memoryOK && !vcoresOK) {
        LOG.warn(
                "Services require {} MB of memory and {} vcores, "
                        + "but the cluster only has {} MB of memory and {} vcores available.",
                requiredMemoryMB, requiredVCores, availableMemoryMB, availableVCores);
    } else if (!memoryOK) {
        LOG.warn("Services require {} MB of memory but the cluster only has {} MB of memory available.",
                requiredMemoryMB, availableMemoryMB);
    } else if (!vcoresOK) {
        LOG.warn("Services require {} vcores but the cluster only has {} vcores available.", requiredVCores,
                availableVCores);
    } else {
        LOG.info("  YARN resources successfully verified.");
    }
}

From source file:co.cask.cdap.operations.yarn.YarnNodes.java

License:Apache License

@Override
public synchronized void collect() throws Exception {
    reset();//from  w ww.j  ava  2 s .  co m
    List<NodeReport> nodeReports;
    YarnClient yarnClient = createYARNClient();
    try {
        nodeReports = yarnClient.getNodeReports();
    } finally {
        yarnClient.stop();
    }
    for (NodeReport nodeReport : nodeReports) {
        switch (nodeReport.getNodeState()) {
        case RUNNING:
            healthyNodes++;
            healthyContainers += nodeReport.getNumContainers();
            break;
        case UNHEALTHY:
        case DECOMMISSIONED:
        case LOST:
            unusableNodes++;
            unusableContainers += nodeReport.getNumContainers();
            break;
        case NEW:
        case REBOOTED:
            newNodes++;
            newContainers += nodeReport.getNumContainers();
            break;
        }
    }
}

From source file:co.cask.cdap.operations.yarn.YarnResources.java

License:Apache License

@Override
public synchronized void collect() throws Exception {
    reset();/* w  w  w. java  2s .  co  m*/
    List<NodeReport> nodeReports;
    YarnClient yarnClient = createYARNClient();
    try {
        nodeReports = yarnClient.getNodeReports();
    } finally {
        yarnClient.stop();
    }
    for (NodeReport nodeReport : nodeReports) {
        NodeId nodeId = nodeReport.getNodeId();
        LOG.debug("Got report for node {}", nodeId);
        if (!nodeReport.getNodeState().isUnusable()) {
            Resource nodeCapability = nodeReport.getCapability();
            Resource nodeUsed = nodeReport.getUsed();

            // some versions of hadoop return null, others do not
            if (nodeCapability != null) {
                LOG.debug("node {} resource capability: memory = {}, vcores = {}", nodeId,
                        nodeCapability.getMemory(), nodeCapability.getVirtualCores());
                totalMemory += nodeCapability.getMemory();
                totalVCores += nodeCapability.getVirtualCores();
            }

            if (nodeUsed != null) {
                LOG.debug("node {} resources used: memory = {}, vcores = {}", nodeId, nodeUsed.getMemory(),
                        nodeUsed.getVirtualCores());
                usedMemory += nodeUsed.getMemory();
                usedVCores += nodeUsed.getVirtualCores();
            }
        }
    }
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void _registerSchedulerAndCreateNMClient(String queue) throws Exception {
    NMTokenCache nmTokenCache = new NMTokenCache();
    nmClient = NMClient.createNMClient();
    nmClient.setNMTokenCache(nmTokenCache);
    nmClient.init(yarnConf);/* w w  w. j  a  va2 s . c om*/
    nmClient.start();
    LOG.debug("Started NMClient, AM '{}' with scheduler for '{}' queue", appId, queue);
    int heartbeatInterval = getConf().getInt(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERNAL_DEFAULT);
    AMRMClient<LlamaContainerRequest> amRmClient = AMRMClient.createAMRMClient();
    amRmClient.setNMTokenCache(nmTokenCache);
    amRmClientAsync = AMRMClientAsync.createAMRMClientAsync(amRmClient, heartbeatInterval,
            YarnRMConnector.this);
    amRmClientAsync.init(yarnConf);
    amRmClientAsync.start();
    String urlWithoutScheme = getConf().get(ADVERTISED_TRACKING_URL_KEY, "http://")
            .substring("http://".length());
    RegisterApplicationMasterResponse response = amRmClientAsync.registerApplicationMaster(
            getConf().get(ADVERTISED_HOSTNAME_KEY, ""), getConf().getInt(ADVERTISED_PORT_KEY, 0),
            urlWithoutScheme);
    maxResource = response.getMaximumResourceCapability();
    nodes = Collections.synchronizedMap(new HashMap<String, Resource>());
    for (NodeReport nodeReport : yarnClient.getNodeReports()) {
        if (nodeReport.getNodeState() == NodeState.RUNNING) {
            String nodeKey = getNodeName(nodeReport.getNodeId());
            nodes.put(nodeKey, nodeReport.getCapability());
            LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey,
                    nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory());
        }
    }
    LOG.debug("Registered with scheduler, AM '{}' for '{}' queue", appId, queue);
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public void onNodesUpdated(List<NodeReport> nodeReports) {
    LOG.debug("Received nodes update for '{}' nodes", nodeReports.size());
    for (NodeReport nodeReport : nodeReports) {
        if (nodeReport.getNodeState() == NodeState.RUNNING) {
            String nodeKey = getNodeName(nodeReport.getNodeId());
            nodes.put(nodeKey, nodeReport.getCapability());
            LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey,
                    nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory());
        } else {//from   w  w  w  . j a  va  2s.  co m
            LOG.debug("Removed node '{}'", nodeReport.getNodeId());
            nodes.remove(getNodeName(nodeReport.getNodeId()));
        }
    }
}

From source file:com.datatorrent.stram.ResourceRequestHandler.java

License:Apache License

/**
 * Tracks update to available resources. Resource availability is used to make decisions about where to request new
 * containers.//from w w  w . j a  v  a 2  s .c om
 *
 * @param nodeReports
 */
public void updateNodeReports(List<NodeReport> nodeReports) {
    // LOG.debug("Got {} updated node reports.", nodeReports.size());
    for (NodeReport nr : nodeReports) {
        StringBuilder sb = new StringBuilder();
        sb.append("rackName=").append(nr.getRackName()).append(",nodeid=").append(nr.getNodeId())
                .append(",numContainers=").append(nr.getNumContainers()).append(",capability=")
                .append(nr.getCapability()).append("used=").append(nr.getUsed()).append("state=")
                .append(nr.getNodeState());
        LOG.info("Node report: " + sb);
        nodeReportMap.put(nr.getNodeId().getHost(), nr);
        nodeToRack.put(nr.getNodeId().getHost(), nr.getRackName());
    }
}

From source file:com.hazelcast.yarn.ApplicationMaster.java

License:Open Source License

@Override
public void onNodesUpdated(List<NodeReport> updated) {
    for (NodeReport node : updated) {
        if (node.getNodeState().isUnusable()) {
            for (HazelcastContainer container : this.containers.values()) {
                if (container.getNodeId().equals(node.getNodeId())) {
                    this.containers.remove(container.getId());
                    LOG.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                            new Object[] { node.getNodeId().getHost(), node.getNodeState() });
                }//from   w  w  w . j a  v  a2s .c  o m
            }

            LOG.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                    new Object[] { node.getNodeId().getHost(), node.getNodeState() });
        }
    }
}

From source file:org.apache.ignite.yarn.ApplicationMaster.java

License:Apache License

/** {@inheritDoc} */
public synchronized void onNodesUpdated(List<NodeReport> updated) {
    for (NodeReport node : updated) {
        // If node unusable.
        if (node.getNodeState().isUnusable()) {
            for (IgniteContainer cont : containers.values()) {
                if (cont.nodeId().equals(node.getNodeId())) {
                    containers.remove(cont.id());

                    log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                            new Object[] { node.getNodeId().getHost(), node.getNodeState() });
                }/* www.  j ava2  s  .  c  om*/
            }

            log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                    new Object[] { node.getNodeId().getHost(), node.getNodeState() });
        }
    }
}

From source file:org.apache.slider.server.appmaster.state.NodeInstance.java

License:Apache License

/**
 * Update the node status.//from  w  w w  .  j  a v a  2s. c o  m
 * The return code is true if the node state changed enough to
 * trigger a re-evaluation of pending requests. That is, either a node
 * became available when it was previously not, or the label changed
 * on an available node.
 *
 * Transitions of a node from live to dead aren't treated as significant,
 * nor label changes on a dead node.
 *
 * @param report latest node report
 * @return true if the node state changed enough for a request evaluation.
 */
public synchronized boolean updateNode(NodeReport report) {
    nodeStateUpdateTime = report.getLastHealthReportTime();
    nodeReport = report;
    NodeState oldState = nodeState;
    boolean oldStateUnusable = oldState.isUnusable();
    nodeState = report.getNodeState();
    boolean newUsable = !nodeState.isUnusable();
    boolean nodeNowAvailable = oldStateUnusable && newUsable;
    String labels = this.nodeLabels;
    nodeLabels = SliderUtils.extractNodeLabel(report);
    return nodeNowAvailable || newUsable && !this.nodeLabels.equals(labels);
}

From source file:org.apache.slider.server.appmaster.state.RoleHistory.java

License:Apache License

/**
 * Update failedNodes and nodemap based on the node state
 * //from  w  w w .j av a2  s . co  m
 * @param updatedNodes list of updated nodes
 */
public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
    for (NodeReport updatedNode : updatedNodes) {
        String hostname = updatedNode.getNodeId() == null ? null : updatedNode.getNodeId().getHost();
        if (hostname == null) {
            continue;
        }
        if (updatedNode.getNodeState() != null && updatedNode.getNodeState().isUnusable()) {
            failedNodes.add(hostname);
            nodemap.remove(hostname);
        } else {
            failedNodes.remove(hostname);
        }
    }
}