Example usage for org.apache.hadoop.yarn.util.resource Resources addTo

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.util.resource Resources addTo.

Prototype

public static Resource addTo(Resource lhs, Resource rhs) 

Source Link

Usage

From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java

License:Apache License

@VisibleForTesting
protected Resource getResourcesUnderUse(RMNodeStatusEvent statusEvent) {
    Resource usedResources = Resource.newInstance(0, 0);
    for (ContainerStatus status : statusEvent.getContainers()) {
        if (containerInUse(status)) {
            RMContainer rmContainer = yarnScheduler.getRMContainer(status.getContainerId());
            // (sdaingade) This check is needed as RMContainer information may not be populated
            // immediately after a RM restart.
            if (rmContainer != null) {
                Resources.addTo(usedResources, rmContainer.getAllocatedResource());
            }// w  w w.j  a va 2 s  .  com
        }
    }
    return usedResources;
}

---

From source file:org.apache.tez.dag.app.rm.DagAwareYarnTaskScheduler.java

License:Apache License

private synchronized List<Assignment> assignNewContainers(List<Container> newContainers, AMState appState,
        boolean isSession) {
    // try to assign the containers as node-local
    List<Assignment> assignments = new ArrayList<>(newContainers.size());
    List<HeldContainer> unassigned = new ArrayList<>(newContainers.size());
    for (Container c : newContainers) {
        HeldContainer hc = new HeldContainer(c);
        heldContainers.put(hc.getId(), hc);
        Resources.addTo(allocatedResources, c.getResource());
        tryAssignNewContainer(hc, hc.getHost(), assignments, unassigned);
    }/*from  ww w. j av a  2  s.  c  om*/

    // try to assign the remaining containers as rack-local
    List<HeldContainer> containers = unassigned;
    unassigned = new ArrayList<>(containers.size());
    for (HeldContainer hc : containers) {
        tryAssignNewContainer(hc, hc.getRack(), assignments, unassigned);
    }

    // try to assign the remaining containers without locality
    containers = unassigned;
    unassigned = new ArrayList<>(containers.size());
    for (HeldContainer hc : containers) {
        tryAssignNewContainer(hc, ResourceRequest.ANY, assignments, unassigned);
    }

    for (HeldContainer hc : unassigned) {
        if (reuseNewContainers) {
            idleTracker.add(hc);
            TaskRequest assigned = tryAssignReuseContainer(hc, appState, isSession);
            if (assigned != null) {
                assignments.add(new Assignment(assigned, hc.getContainer()));
            }
        } else {
            releaseContainer(hc);
        }
    }

    return assignments;
}

---

From source file:org.apache.tez.dag.app.rm.TaskScheduler.java

License:Apache License

private void assignContainer(Object task, Container container, CookieContainerRequest assigned) {
    CookieContainerRequest request = removeTaskRequest(task);
    assert request != null;
    //assert assigned.equals(request);

    Container result = taskAllocations.put(task, container);
    assert result == null;
    containerAssigments.put(container.getId(), task);

    Resources.addTo(allocatedResources, container.getResource());
}

---

From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java

License:Apache License

private void assignContainer(Object task, Container container, CookieContainerRequest assigned) {
    CookieContainerRequest request = removeTaskRequest(task);
    assert request != null;
    //assert assigned.equals(request);

    Container result = taskAllocations.put(task, container);
    assert result == null;
    inUseContainers.add(container.getId());
    containerAssignments.put(container.getId(), task);
    HeldContainer heldContainer = heldContainers.get(container.getId());
    if (!shouldReuseContainers && heldContainer == null) {
        heldContainers.put(container.getId(), new HeldContainer(container, -1, -1, assigned));
        Resources.addTo(allocatedResources, container.getResource());
    } else {/*from w  ww .j  ava  2  s. c om*/
        if (heldContainer.isNew()) {
            // check for existence before adding since the first container potentially
            // has the broadest signature as subsequent uses dont expand any dimension.
            // This will need to be enhanced to track other signatures too when we
            // think about preferring within vertex matching etc.
            heldContainers.put(container.getId(), new HeldContainer(container,
                    heldContainer.getNextScheduleTime(), heldContainer.getContainerExpiryTime(), assigned));
        }
        heldContainer.setLastTaskInfo(assigned);
    }
}

---

From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java

License:Apache License

private void pushNewContainerToDelayed(List<Container> containers) {
    long expireTime = -1;
    if (idleContainerTimeoutMin > 0) {
        long currentTime = System.currentTimeMillis();
        expireTime = currentTime + idleContainerTimeoutMin;
    }//from   w  ww  .ja v  a  2 s.co  m

    synchronized (delayedContainerManager) {
        for (Container container : containers) {
            if (heldContainers.put(container.getId(),
                    new HeldContainer(container, -1, expireTime, null)) != null) {
                throw new TezUncheckedException("New container " + container.getId() + " is already held.");
            }
            long nextScheduleTime = delayedContainerManager.maxScheduleTimeSeen;
            if (delayedContainerManager.maxScheduleTimeSeen == -1) {
                nextScheduleTime = System.currentTimeMillis();
            }
            Resources.addTo(allocatedResources, container.getResource());
            delayedContainerManager.addDelayedContainer(container, nextScheduleTime + 1);
        }
    }
    delayedContainerManager.triggerScheduling(false);
}

---