Example usage for org.apache.hadoop.yarn.api.records ResourceRequest getNumContainers

List of usage examples for org.apache.hadoop.yarn.api.records ResourceRequest getNumContainers

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ResourceRequest
  6. getNumContainers

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ResourceRequest getNumContainers.

Prototype

@Public
@Stable
public abstract int getNumContainers();

Source Link

Document

Get the number of containers required with the given specifications.

Usage

From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

private void addResourceRequest(Priority priority, String resourceName, Resource capability,
        int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);
    if (remoteRequests == null) {
        remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
        this.remoteRequestsTable.put(priority, remoteRequests);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Added priority=" + priority);
        }//from  w ww  .java  2  s.  c o  m
    }
    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        reqMap = new HashMap<Resource, ResourceRequest>();
        remoteRequests.put(resourceName, reqMap);
    }
    ResourceRequest remoteRequest = reqMap.get(capability);
    if (remoteRequest == null) {
        remoteRequest = BuilderUtils.newResourceRequest(priority, resourceName, capability, 0);
        reqMap.put(capability, remoteRequest);
    }

    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + containerCount);

    // Note this down for next interaction with ResourceManager
    addResourceRequestToAsk(remoteRequest);

    if (LOG.isDebugEnabled()) {
        LOG.debug("addResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

private void decResourceRequest(Priority priority, String resourceName, Resource capability,
        int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);

    if (remoteRequests == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as priority " + priority + " is not present in request table");
        }//  www . j ava  2s. co  m
        return;
    }

    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as " + resourceName + " is not present in request table");
        }
        return;
    }
    ResourceRequest remoteRequest = reqMap.get(capability);

    if (LOG.isDebugEnabled()) {
        LOG.debug("BEFORE decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }

    remoteRequest.setNumContainers(remoteRequest.getNumContainers() - containerCount);
    if (remoteRequest.getNumContainers() < 0) {
        // guard against spurious removals
        remoteRequest.setNumContainers(0);
    }
    // send the ResourceRequest to RM even if is 0 because it needs to override
    // a previously sent value. If ResourceRequest was not sent previously then
    // sending 0 aught to be a no-op on RM
    addResourceRequestToAsk(remoteRequest);

    // delete entries from map if no longer needed
    if (remoteRequest.getNumContainers() == 0) {
        reqMap.remove(capability);
        if (reqMap.size() == 0) {
            remoteRequests.remove(resourceName);
        }
        if (remoteRequests.size() == 0) {
            remoteRequestsTable.remove(priority);
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.info("AFTER decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java

License:Apache License

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();

    try {//from  w w  w  . ja  va 2s .com
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask.size());
            for (ResourceRequest r : ask) {
                // create a copy of ResourceRequest as we might change it while the 
                // RPC layer is using it to send info across
                askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(),
                        r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression()));
            }
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();

            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);

            ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null)
                    ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove)
                    : null;

            allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList,
                    releaseList, blacklistRequest);
            // clear blacklistAdditions and blacklistRemovals before 
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }

        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) {
                    for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) {
                        for (ResourceRequestInfo request : capabalities.values()) {
                            addResourceRequestToAsk(request.remoteRequest);
                        }
                    }
                }
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }

        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // we hit an exception in allocate()
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // requests could have been added or deleted during call to allocate
                // If requests were added/removed then there is nothing to do since
                // the ResourceRequest object in ask would have the actual new value.
                // If ask does not have this ResourceRequest then it was unchanged and
                // so we can add the value back safely.
                // This assumes that there will no concurrent calls to allocate() and
                // so we dont have to worry about ask being changed in the
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }

                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java

License:Apache License

private void setupAsk(AbstractProcess proc) {
    ContainerSpecification cSpec = proc.getContainerSpecification();
    ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);

    rsrcRequest.setHostName(cSpec.getHostname());

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);/*w w w  .  j a va2 s.  c  om*/
    rsrcRequest.setPriority(pri);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(cSpec.getMemory());
    rsrcRequest.setCapability(capability);

    rsrcRequest.setNumContainers(1);

    AskRecord ar = new AskRecord();
    ar.req = rsrcRequest;
    ar.proc = proc;

    Set<AskRecord> arSet = resource2AskMap.get(capability);
    if (arSet == null) {
        arSet = new HashSet<AskRecord>();
        resource2AskMap.put(capability, arSet);
    }
    arSet.add(ar);
    proc2AskMap.put(proc, ar);

    System.err.println(proc + " -> [" + rsrcRequest.getHostName() + ", " + rsrcRequest.getNumContainers() + ", "
            + rsrcRequest.getPriority() + ", " + rsrcRequest.getCapability().getMemory() + "]");

    asks.add(rsrcRequest);
}

From source file:org.apache.twill.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

private void decResourceRequest(Priority priority, String resourceName, Resource capability,
        int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);

    if (remoteRequests == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as priority " + priority + " is not present in request table");
        }//  www  .j a  v a  2 s  .com
        return;
    }

    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as " + resourceName + " is not present in request table");
        }
        return;
    }
    ResourceRequest remoteRequest = reqMap.get(capability);

    if (LOG.isDebugEnabled()) {
        LOG.debug("BEFORE decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }

    remoteRequest.setNumContainers(remoteRequest.getNumContainers() - containerCount);
    if (remoteRequest.getNumContainers() < 0) {
        // guard against spurious removals
        remoteRequest.setNumContainers(0);
    }
    // Send the ResourceRequest to RM even if is 0 because it needs to override
    // a previously sent value. If ResourceRequest was not sent previously then
    // sending 0 ought to be a no-op on RM.
    addResourceRequestToAsk(remoteRequest);

    // Delete entries from map if no longer needed.
    if (remoteRequest.getNumContainers() == 0) {
        reqMap.remove(capability);
        if (reqMap.size() == 0) {
            remoteRequests.remove(resourceName);
        }
        if (remoteRequests.size() == 0) {
            remoteRequestsTable.remove(priority);
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.info("AFTER decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

From source file:org.deeplearning4j.iterativereduce.runtime.Utils.java

License:Apache License

public static ResourceRequest createResourceRequest(String host, int amount, int memory) {
    ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);
    rsrcRequest.setHostName(host);//from   ww  w .  j  a v a2s .co  m

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);
    rsrcRequest.setPriority(pri);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    rsrcRequest.setCapability(capability);

    rsrcRequest.setNumContainers(amount);

    LOG.debug("Created a resource request" + ", host=" + rsrcRequest.getHostName() + ", memory="
            + rsrcRequest.getCapability().getMemory() + ", amount=" + rsrcRequest.getNumContainers()
            + ", priority=" + rsrcRequest.getPriority().getPriority());

    return rsrcRequest;
}

From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java

License:Apache License

/**
 * Changed the return type to AllocateResponse which use to hold a reference to 
 * AMResponse. //from w  w w  .  j  a v a 2 s .c o  m
 * 
 * AMResponse seems to have disappeared in CDH 4.6
 * 
 * @param requestedContainers
 * @param releasedContainers
 * @return
 * @throws YarnRemoteException
 */

public AllocateResponse allocateRequest(List<ResourceRequest> requestedContainers,
        List<ContainerId> releasedContainers) throws YarnRemoteException {

    if (amResourceManager == null)
        throw new IllegalStateException(
                "Cannot send allocation request before connecting to the resource manager!");

    LOG.info("Sending allocation request" + ", requestedSize=" + requestedContainers.size() + ", releasedSize="
            + releasedContainers.size());

    for (ResourceRequest req : requestedContainers)
        LOG.info("Requesting container, host=" + req.getHostName() + ", amount=" + req.getNumContainers()
                + ", memory=" + req.getCapability().getMemory() + ", priority="
                + req.getPriority().getPriority());

    for (ContainerId rel : releasedContainers)
        LOG.info("Releasing container: " + rel.getId());

    AllocateRequest request = Records.newRecord(AllocateRequest.class);
    request.setResponseId(rmRequestId.incrementAndGet());
    request.setApplicationAttemptId(appAttemptId);
    request.addAllAsks(requestedContainers);
    request.addAllReleases(releasedContainers);

    AllocateResponse response = amResourceManager.allocate(request);

    //response.getAllocatedContainers()

    LOG.debug("Got an allocation response, " + ", responseId=" + response.getResponseId() + ", numClusterNodes="
            + response.getNumClusterNodes() + ", headroom=" + response.getAvailableResources().getMemory()
            + ", allocatedSize=" + response.getAllocatedContainers().size() + ", updatedNodes="
            + response.getUpdatedNodes().size() + ", reboot=" + response.getReboot() + ", completedSize="
            + response.getCompletedContainersStatuses().size());

    return response;
}

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocator.java

License:Apache License

@Override
protected AllocateResponse doContainerRequest() {
    List<ResourceRequest> requestedContainers = new ArrayList<ResourceRequest>();
    Map<String, Integer> allocateCounts = allocateCountTracker.getAllocateCounts();

    for (Entry<String, Integer> entry : allocateCounts.entrySet()) {
        requestedContainers.add(getContainerResourceRequest(entry.getValue(), entry.getKey()));
    }//  w w w.  j  a  va 2s.c  o  m

    // add pending containers to be released
    List<ContainerId> release = new ArrayList<ContainerId>();
    ContainerId element = null;
    while ((element = releaseContainers.poll()) != null) {
        release.add(element);
    }

    if (log.isDebugEnabled()) {
        log.debug("Requesting containers using " + requestedContainers.size() + " requests.");
        for (ResourceRequest resourceRequest : requestedContainers) {
            log.debug(
                    "ResourceRequest: " + resourceRequest + " with count=" + resourceRequest.getNumContainers()
                            + " with hostName=" + resourceRequest.getResourceName());
        }
        log.debug("Releasing containers " + release.size());
        for (ContainerId cid : release) {
            log.debug("Release container=" + cid);
        }
        log.debug("Request id will be: " + requestId.get());
    }

    // build the allocation request
    AllocateRequest request = Records.newRecord(AllocateRequest.class);
    request.setResponseId(requestId.get());
    request.setAskList(requestedContainers);
    request.setReleaseList(release);
    request.setProgress(applicationProgress);

    // do request and return response
    AllocateResponse allocate = getRmTemplate().allocate(request);
    requestId.set(allocate.getResponseId());
    return allocate;
}

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

@Test
public void testLegacyAnyCount() throws Exception {
    DefaultContainerAllocator allocator = new DefaultContainerAllocator();
    allocator.setConfiguration(new Configuration());
    allocator.setLocality(false);/*from   w w w  .  jav a  2s  . c  o  m*/
    TestUtils.callMethod("internalInit", allocator);

    allocator.allocateContainers(2);

    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));

    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
}

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java

License:Apache License

@Test
public void testAnyHostNoLocalityRequests() throws Exception {
    DefaultContainerAllocator allocator = new DefaultContainerAllocator();
    allocator.setConfiguration(new Configuration());
    allocator.setLocality(false);/*from w ww.j a  v  a  2  s .  co m*/
    TestUtils.callMethod("internalInit", allocator);

    ContainerAllocateData data = new ContainerAllocateData();
    data.addAny(2);
    allocator.allocateContainers(data);

    List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator);
    assertThat(createRequests, notNullValue());
    assertThat(createRequests.size(), is(1));

    ResourceRequest req0 = createRequests.get(0);
    assertThat(req0.getResourceName(), is("*"));
    assertThat(req0.getPriority().getPriority(), is(0));
    assertThat(req0.getNumContainers(), is(2));
    assertThat(req0.getRelaxLocality(), is(true));
}