Example usage for org.apache.hadoop.yarn.util.resource Resources add

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.util.resource Resources add.

Prototype

public static Resource add(Resource lhs, Resource rhs) 

Source Link

Usage

From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java

License:Apache License

@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
    if (!(event instanceof RMNodeStatusEvent)) {
        logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
        return;/*from   w w w  . j  av a 2 s  .  c  om*/
    }

    RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
    RMNode rmNode = context.getRMNodes().get(event.getNodeId());
    String hostName = rmNode.getNodeID().getHost();

    Node host = nodeStore.getNode(hostName);
    if (host != null) {
        host.snapshotRunningContainers();
    }

    /*
     * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
     * If the sum is greater than the max capacity of the node, reject the offer.
     */
    Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
    Resource currentResources = getResourcesUnderUse(statusEvent);

    if (offerWithinResourceLimits(currentResources, offeredResources)) {
        yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
        logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(),
                offeredResources.getVirtualCores(), offeredResources.getMemory());
    } else {
        logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory",
                rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
    }
}

---

From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java

License:Apache License

/**
 * Checks if any containers were allocated in the current scheduler run and
 * launches the corresponding Mesos tasks. It also updates the node
 * capacity depending on what portion of the consumed offers were actually
 * used.//from  w  ww  .j a v a2 s  .c  o  m
 */
@VisibleForTesting
protected void handleContainerAllocation(RMNode rmNode) {
    String host = rmNode.getNodeID().getHost();

    ConsumedOffer consumedOffer = offerLifecycleMgr.drainConsumedOffer(host);
    if (consumedOffer == null) {
        LOGGER.debug("No offer consumed for {}", host);
        return;
    }

    Node node = nodeStore.getNode(host);
    Set<RMContainer> containersBeforeSched = node.getContainerSnapshot();
    Set<RMContainer> containersAfterSched = new HashSet<>(node.getNode().getRunningContainers());

    Set<RMContainer> containersAllocatedByMesosOffer = (containersBeforeSched == null) ? containersAfterSched
            : Sets.difference(containersAfterSched, containersBeforeSched);

    if (containersAllocatedByMesosOffer.isEmpty()) {
        LOGGER.debug("No containers allocated using Mesos offers for host: {}", host);
        for (Protos.Offer offer : consumedOffer.getOffers()) {
            offerLifecycleMgr.declineOffer(offer);
        }
        decrementNodeCapacity(rmNode, OfferUtils.getYarnResourcesFromMesosOffers(consumedOffer.getOffers()));
    } else {
        LOGGER.debug("Containers allocated using Mesos offers for host: {} count: {}", host,
                containersAllocatedByMesosOffer.size());

        // Identify the Mesos tasks that need to be launched
        List<Protos.TaskInfo> tasks = Lists.newArrayList();
        Resource resUsed = Resource.newInstance(0, 0);

        for (RMContainer newContainer : containersAllocatedByMesosOffer) {
            tasks.add(getTaskInfoForContainer(newContainer, consumedOffer, node));
            resUsed = Resources.add(resUsed, newContainer.getAllocatedResource());
        }

        // Reduce node capacity to account for unused offers
        Resource resOffered = OfferUtils.getYarnResourcesFromMesosOffers(consumedOffer.getOffers());
        Resource resUnused = Resources.subtract(resOffered, resUsed);
        decrementNodeCapacity(rmNode, resUnused);
        myriadDriver.getDriver().launchTasks(consumedOffer.getOfferIds(), tasks);
    }

    // No need to hold on to the snapshot anymore
    node.removeContainerSnapshot();
}

---

From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java

License:Apache License

/**
 * Increments the capacity for the specified RMNode
 * /*  w w w. j  a  va2  s .  com*/
 * @param rmNode
 * @param removedCapacity
 */
public void incrementNodeCapacity(RMNode rmNode, Resource addedCapacity) {
    setNodeCapacity(rmNode, Resources.add(rmNode.getTotalCapability(), addedCapacity));
}

---