Example usage for org.apache.hadoop.yarn.api.records ResourceRequest getRelaxLocality

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ResourceRequest getRelaxLocality.

Prototype

@Public
@Stable
public abstract boolean getRelaxLocality();

Source Link

Document

Get whether locality relaxation is enabled with this ResourceRequest.

Usage

From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java

License:Apache License

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();

    try {/*from  www .j  av a 2 s  . c  o m*/
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask.size());
            for (ResourceRequest r : ask) {
                // create a copy of ResourceRequest as we might change it while the 
                // RPC layer is using it to send info across
                askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(),
                        r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression()));
            }
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();

            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);

            ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null)
                    ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove)
                    : null;

            allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList,
                    releaseList, blacklistRequest);
            // clear blacklistAdditions and blacklistRemovals before 
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }

        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) {
                    for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) {
                        for (ResourceRequestInfo request : capabalities.values()) {
                            addResourceRequestToAsk(request.remoteRequest);
                        }
                    }
                }
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }

        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // we hit an exception in allocate()
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // requests could have been added or deleted during call to allocate
                // If requests were added/removed then there is nothing to do since
                // the ResourceRequest object in ask would have the actual new value.
                // If ask does not have this ResourceRequest then it was unchanged and
                // so we can add the value back safely.
                // This assumes that there will no concurrent calls to allocate() and
                // so we dont have to worry about ask being changed in the
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }

                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

@Test
public void testLegacyAnyCount() throws Exception {
    DefaultContainerAllocator allocator = new DefaultContainerAllocator();
    allocator.setConfiguration(new Configuration());
    allocator.setLocality(false);//  ww w . j  a v  a  2s.c o  m
    TestUtils.callMethod("internalInit", allocator);

    allocator.allocateContainers(2);

    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));

    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

@Test
public void testAnyHostNoLocalityRequests() throws Exception {
    DefaultContainerAllocator allocator = new DefaultContainerAllocator();
    allocator.setConfiguration(new Configuration());
    allocator.setLocality(false);// w  w  w . j a  v a2  s .  c  o  m
    TestUtils.callMethod("internalInit", allocator);

    ContainerAllocateData data = new ContainerAllocateData();
    data.addAny(2);
    allocator.allocateContainers(data);

    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));

    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

private static void assertResourceRequest(ResourceRequest req, String resourceName, Integer priority,
        Integer numContainers, Boolean relaxLocality, Integer capMemory, Integer capCores) {
    if (resourceName != null) {
        assertThat(req.getResourceName(), is(resourceName));
    }/* w  w  w . j  a  va2  s . co  m*/
    if (priority != null) {
        assertThat(req.getPriority().getPriority(), is(priority));
    }
    if (numContainers != null) {
        assertThat(req.getNumContainers(), is(numContainers));
    }
    if (relaxLocality != null) {
        assertThat(req.getRelaxLocality(), is(relaxLocality));
    }
    if (capMemory != null) {
        assertThat(req.getCapability().getMemory(), is(capMemory));
    }
    if (capCores != null) {
        assertThat(req.getCapability().getVirtualCores(), is(capCores));
    }
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

private static ResourceRequest matchRequest(List<ResourceRequest> createRequests, String name, int priority,
        int containers, boolean relax) {
    for (ResourceRequest r : createRequests) {
        if (r.getResourceName().equals(name) && r.getPriority().getPriority() == priority
                && r.getNumContainers() == containers && r.getRelaxLocality() == relax) {
            return r;
        }/* w  w w . j  a  v  a 2s .  c om*/
    }
    return null;
}

---