Example usage for org.apache.hadoop.yarn.api.records ResourceRequest getResourceName

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ResourceRequest getResourceName.

Prototype

@Public
@Stable
public abstract String getResourceName();

Source Link

Document

Get the resource (e.g.

Usage

From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java

License:Apache License

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();

    try {/*from  w ww . j  av a2 s  .co m*/
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask.size());
            for (ResourceRequest r : ask) {
                // create a copy of ResourceRequest as we might change it while the 
                // RPC layer is using it to send info across
                askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(),
                        r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression()));
            }
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();

            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);

            ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null)
                    ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove)
                    : null;

            allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList,
                    releaseList, blacklistRequest);
            // clear blacklistAdditions and blacklistRemovals before 
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }

        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) {
                    for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) {
                        for (ResourceRequestInfo request : capabalities.values()) {
                            addResourceRequestToAsk(request.remoteRequest);
                        }
                    }
                }
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }

        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // we hit an exception in allocate()
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // requests could have been added or deleted during call to allocate
                // If requests were added/removed then there is nothing to do since
                // the ResourceRequest object in ask would have the actual new value.
                // If ask does not have this ResourceRequest then it was unchanged and
                // so we can add the value back safely.
                // This assumes that there will no concurrent calls to allocate() and
                // so we dont have to worry about ask being changed in the
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }

                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocator.java

License:Apache License

@Override
protected AllocateResponse doContainerRequest() {
    List<ResourceRequest> requestedContainers = new ArrayList<ResourceRequest>();
    Map<String, Integer> allocateCounts = allocateCountTracker.getAllocateCounts();

    for (Entry<String, Integer> entry : allocateCounts.entrySet()) {
        requestedContainers.add(getContainerResourceRequest(entry.getValue(), entry.getKey()));
    }//w  ww  .  j a v  a 2  s  . co  m

    // add pending containers to be released
    List<ContainerId> release = new ArrayList<ContainerId>();
    ContainerId element = null;
    while ((element = releaseContainers.poll()) != null) {
        release.add(element);
    }

    if (log.isDebugEnabled()) {
        log.debug("Requesting containers using " + requestedContainers.size() + " requests.");
        for (ResourceRequest resourceRequest : requestedContainers) {
            log.debug(
                    "ResourceRequest: " + resourceRequest + " with count=" + resourceRequest.getNumContainers()
                            + " with hostName=" + resourceRequest.getResourceName());
        }
        log.debug("Releasing containers " + release.size());
        for (ContainerId cid : release) {
            log.debug("Release container=" + cid);
        }
        log.debug("Request id will be: " + requestId.get());
    }

    // build the allocation request
    AllocateRequest request = Records.newRecord(AllocateRequest.class);
    request.setResponseId(requestId.get());
    request.setAskList(requestedContainers);
    request.setReleaseList(release);
    request.setProgress(applicationProgress);

    // do request and return response
    AllocateResponse allocate = getRmTemplate().allocate(request);
    requestId.set(allocate.getResponseId());
    return allocate;
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

@Test
public void testLegacyAnyCount() throws Exception {
    DefaultContainerAllocator allocator = new DefaultContainerAllocator();
    allocator.setConfiguration(new Configuration());
    allocator.setLocality(false);/*from  w  w  w  .j a v a  2  s .  co m*/
    TestUtils.callMethod("internalInit", allocator);

    allocator.allocateContainers(2);

    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));

    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

@Test
public void testAnyHostNoLocalityRequests() throws Exception {
    DefaultContainerAllocator allocator = new DefaultContainerAllocator();
    allocator.setConfiguration(new Configuration());
    allocator.setLocality(false);/*from w w w.j  av  a 2s .c o m*/
    TestUtils.callMethod("internalInit", allocator);

    ContainerAllocateData data = new ContainerAllocateData();
    data.addAny(2);
    allocator.allocateContainers(data);

    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));

    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

private static void assertResourceRequest(ResourceRequest req, String resourceName, Integer priority,
        Integer numContainers, Boolean relaxLocality, Integer capMemory, Integer capCores) {
    if (resourceName != null) {
        assertThat(req.getResourceName(), is(resourceName));
    }/* ww  w. ja  va2 s .c o  m*/
    if (priority != null) {
        assertThat(req.getPriority().getPriority(), is(priority));
    }
    if (numContainers != null) {
        assertThat(req.getNumContainers(), is(numContainers));
    }
    if (relaxLocality != null) {
        assertThat(req.getRelaxLocality(), is(relaxLocality));
    }
    if (capMemory != null) {
        assertThat(req.getCapability().getMemory(), is(capMemory));
    }
    if (capCores != null) {
        assertThat(req.getCapability().getVirtualCores(), is(capCores));
    }
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

private static ResourceRequest matchRequest(List<ResourceRequest> createRequests, String name, int priority,
        int containers, boolean relax) {
    for (ResourceRequest r : createRequests) {
        if (r.getResourceName().equals(name) && r.getPriority().getPriority() == priority
                && r.getNumContainers() == containers && r.getRelaxLocality() == relax) {
            return r;
        }//from ww w .j av a  2s.  c o  m
    }
    return null;
}

---