Example usage for org.apache.hadoop.yarn.api.records ResourceRequest getCapability

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ResourceRequest getCapability.

Prototype

@Public
@Stable
public abstract Resource getCapability();

Source Link

Document

Get the Resource capability of the request.

Usage

From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java

License:Apache License

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();

    try {/*from  w  ww .ja va 2 s .c  om*/
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask.size());
            for (ResourceRequest r : ask) {
                // create a copy of ResourceRequest as we might change it while the 
                // RPC layer is using it to send info across
                askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(),
                        r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression()));
            }
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();

            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);

            ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null)
                    ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove)
                    : null;

            allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList,
                    releaseList, blacklistRequest);
            // clear blacklistAdditions and blacklistRemovals before 
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }

        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) {
                    for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) {
                        for (ResourceRequestInfo request : capabalities.values()) {
                            addResourceRequestToAsk(request.remoteRequest);
                        }
                    }
                }
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }

        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // we hit an exception in allocate()
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // requests could have been added or deleted during call to allocate
                // If requests were added/removed then there is nothing to do since
                // the ResourceRequest object in ask would have the actual new value.
                // If ask does not have this ResourceRequest then it was unchanged and
                // so we can add the value back safely.
                // This assumes that there will no concurrent calls to allocate() and
                // so we dont have to worry about ask being changed in the
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }

                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

---

From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java

License:Apache License

private void setupAsk(AbstractProcess proc) {
    ContainerSpecification cSpec = proc.getContainerSpecification();
    ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);

    rsrcRequest.setHostName(cSpec.getHostname());

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);/*from   w w  w.  ja  v a  2s  . c  om*/
    rsrcRequest.setPriority(pri);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(cSpec.getMemory());
    rsrcRequest.setCapability(capability);

    rsrcRequest.setNumContainers(1);

    AskRecord ar = new AskRecord();
    ar.req = rsrcRequest;
    ar.proc = proc;

    Set<AskRecord> arSet = resource2AskMap.get(capability);
    if (arSet == null) {
        arSet = new HashSet<AskRecord>();
        resource2AskMap.put(capability, arSet);
    }
    arSet.add(ar);
    proc2AskMap.put(proc, ar);

    System.err.println(proc + " -> [" + rsrcRequest.getHostName() + ", " + rsrcRequest.getNumContainers() + ", "
            + rsrcRequest.getPriority() + ", " + rsrcRequest.getCapability().getMemory() + "]");

    asks.add(rsrcRequest);
}

---

From source file:org.deeplearning4j.iterativereduce.runtime.Utils.java

License:Apache License

public static ResourceRequest createResourceRequest(String host, int amount, int memory) {
    ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);
    rsrcRequest.setHostName(host);/*from   w  w w. ja v  a 2 s.  c  o  m*/

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);
    rsrcRequest.setPriority(pri);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    rsrcRequest.setCapability(capability);

    rsrcRequest.setNumContainers(amount);

    LOG.debug("Created a resource request" + ", host=" + rsrcRequest.getHostName() + ", memory="
            + rsrcRequest.getCapability().getMemory() + ", amount=" + rsrcRequest.getNumContainers()
            + ", priority=" + rsrcRequest.getPriority().getPriority());

    return rsrcRequest;
}

---

From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java

License:Apache License

/**
 * Changed the return type to AllocateResponse which use to hold a reference to 
 * AMResponse. /* w  w w.ja v  a  2s .  c  o m*/
 * 
 * AMResponse seems to have disappeared in CDH 4.6
 * 
 * @param requestedContainers
 * @param releasedContainers
 * @return
 * @throws YarnRemoteException
 */

public AllocateResponse allocateRequest(List<ResourceRequest> requestedContainers,
        List<ContainerId> releasedContainers) throws YarnRemoteException {

    if (amResourceManager == null)
        throw new IllegalStateException(
                "Cannot send allocation request before connecting to the resource manager!");

    LOG.info("Sending allocation request" + ", requestedSize=" + requestedContainers.size() + ", releasedSize="
            + releasedContainers.size());

    for (ResourceRequest req : requestedContainers)
        LOG.info("Requesting container, host=" + req.getHostName() + ", amount=" + req.getNumContainers()
                + ", memory=" + req.getCapability().getMemory() + ", priority="
                + req.getPriority().getPriority());

    for (ContainerId rel : releasedContainers)
        LOG.info("Releasing container: " + rel.getId());

    AllocateRequest request = Records.newRecord(AllocateRequest.class);
    request.setResponseId(rmRequestId.incrementAndGet());
    request.setApplicationAttemptId(appAttemptId);
    request.addAllAsks(requestedContainers);
    request.addAllReleases(releasedContainers);

    AllocateResponse response = amResourceManager.allocate(request);

    //response.getAllocatedContainers()

    LOG.debug("Got an allocation response, " + ", responseId=" + response.getResponseId() + ", numClusterNodes="
            + response.getNumClusterNodes() + ", headroom=" + response.getAvailableResources().getMemory()
            + ", allocatedSize=" + response.getAllocatedContainers().size() + ", updatedNodes="
            + response.getUpdatedNodes().size() + ", reboot=" + response.getReboot() + ", completedSize="
            + response.getCompletedContainersStatuses().size());

    return response;
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

private static void assertResourceRequest(ResourceRequest req, String resourceName, Integer priority,
        Integer numContainers, Boolean relaxLocality, Integer capMemory, Integer capCores) {
    if (resourceName != null) {
        assertThat(req.getResourceName(), is(resourceName));
    }/*from   ww  w . j a  v  a  2s.c o  m*/
    if (priority != null) {
        assertThat(req.getPriority().getPriority(), is(priority));
    }
    if (numContainers != null) {
        assertThat(req.getNumContainers(), is(numContainers));
    }
    if (relaxLocality != null) {
        assertThat(req.getRelaxLocality(), is(relaxLocality));
    }
    if (capMemory != null) {
        assertThat(req.getCapability().getMemory(), is(capMemory));
    }
    if (capCores != null) {
        assertThat(req.getCapability().getVirtualCores(), is(capCores));
    }
}

---