Example usage for org.apache.hadoop.yarn.api.protocolrecords AllocateRequest newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.protocolrecords AllocateRequest newInstance.

Prototype

@Public
    @Stable
    public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk,
            List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest) 

Source Link

Usage

From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java

License:Apache License

@Override
public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();

    try {//from w  ww  .  ja  va 2s .  com
        synchronized (this) {
            askList = new ArrayList<ResourceRequest>(ask.size());
            for (ResourceRequest r : ask) {
                // create a copy of ResourceRequest as we might change it while the 
                // RPC layer is using it to send info across
                askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(),
                        r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression()));
            }
            releaseList = new ArrayList<ContainerId>(release);
            // optimistically clear this collection assuming no RPC failure
            ask.clear();
            release.clear();

            blacklistToAdd.addAll(blacklistAdditions);
            blacklistToRemove.addAll(blacklistRemovals);

            ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null)
                    ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove)
                    : null;

            allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList,
                    releaseList, blacklistRequest);
            // clear blacklistAdditions and blacklistRemovals before 
            // unsynchronized part
            blacklistAdditions.clear();
            blacklistRemovals.clear();
        }

        try {
            allocateResponse = rmClient.allocate(allocateRequest);
        } catch (ApplicationMasterNotRegisteredException e) {
            LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
            synchronized (this) {
                release.addAll(this.pendingRelease);
                blacklistAdditions.addAll(this.blacklistedNodes);
                for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) {
                    for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) {
                        for (ResourceRequestInfo request : capabalities.values()) {
                            addResourceRequestToAsk(request.remoteRequest);
                        }
                    }
                }
            }
            // re register with RM
            registerApplicationMaster();
            allocateResponse = allocate(progressIndicator);
            return allocateResponse;
        }

        synchronized (this) {
            // update these on successful RPC
            clusterNodeCount = allocateResponse.getNumClusterNodes();
            lastResponseId = allocateResponse.getResponseId();
            clusterAvailableResources = allocateResponse.getAvailableResources();
            if (!allocateResponse.getNMTokens().isEmpty()) {
                populateNMTokens(allocateResponse.getNMTokens());
            }
            if (allocateResponse.getAMRMToken() != null) {
                updateAMRMToken(allocateResponse.getAMRMToken());
            }
            if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
                removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
            }
        }
    } finally {
        // TODO how to differentiate remote yarn exception vs error in rpc
        if (allocateResponse == null) {
            // we hit an exception in allocate()
            // preserve ask and release for next call to allocate()
            synchronized (this) {
                release.addAll(releaseList);
                // requests could have been added or deleted during call to allocate
                // If requests were added/removed then there is nothing to do since
                // the ResourceRequest object in ask would have the actual new value.
                // If ask does not have this ResourceRequest then it was unchanged and
                // so we can add the value back safely.
                // This assumes that there will no concurrent calls to allocate() and
                // so we dont have to worry about ask being changed in the
                // synchronized block at the beginning of this method.
                for (ResourceRequest oldAsk : askList) {
                    if (!ask.contains(oldAsk)) {
                        ask.add(oldAsk);
                    }
                }

                blacklistAdditions.addAll(blacklistToAdd);
                blacklistRemovals.addAll(blacklistToRemove);
            }
        }
    }
    return allocateResponse;
}

---

From source file:org.apache.hama.bsp.JobImpl.java

License:Apache License

@Override
public JobState startJob() throws Exception {

    this.allocatedContainers = new ArrayList<Container>(numBSPTasks);
    NMTokenCache nmTokenCache = new NMTokenCache();
    while (allocatedContainers.size() < numBSPTasks) {
        AllocateRequest req = AllocateRequest.newInstance(lastResponseID, 0.0f,
                createBSPTaskRequest(numBSPTasks - allocatedContainers.size(), taskMemoryInMb, priority),
                releasedContainers, null);

        AllocateResponse allocateResponse = resourceManager.allocate(req);
        for (NMToken token : allocateResponse.getNMTokens()) {
            nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
        }// w  w w  .  j a  v a2 s .  com

        LOG.info("Got response ID: " + allocateResponse.getResponseId() + " with num of containers: "
                + allocateResponse.getAllocatedContainers().size() + " and following resources: "
                + allocateResponse.getAvailableResources().getMemory() + "mb");
        this.lastResponseID = allocateResponse.getResponseId();

        this.allocatedContainers.addAll(allocateResponse.getAllocatedContainers());

        LOG.info("Waiting to allocate " + (numBSPTasks - allocatedContainers.size()) + " more containers...");

        Thread.sleep(1000l);
    }

    LOG.info("Got " + allocatedContainers.size() + " containers!");

    int id = 0;
    for (Container allocatedContainer : allocatedContainers) {
        LOG.info("Launching task on a new container." + ", containerId=" + allocatedContainer.getId()
                + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":"
                + allocatedContainer.getNodeId().getPort() + ", containerNodeURI="
                + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory"
                + allocatedContainer.getResource().getMemory());

        // Connect to ContainerManager on the allocated container
        String user = conf.get("bsp.user.name");
        if (user == null) {
            user = System.getenv(ApplicationConstants.Environment.USER.name());
        }

        ContainerManagementProtocol cm = null;
        try {
            cm = getContainerManagementProtocolProxy(yarnRPC,
                    nmTokenCache.getToken(allocatedContainer.getNodeId().toString()),
                    allocatedContainer.getNodeId(), user);
        } catch (Exception e) {
            LOG.error("Failed to create ContainerManager...");
            if (cm != null)
                yarnRPC.stopProxy(cm, conf);
            e.printStackTrace();
        }

        BSPTaskLauncher runnableLaunchContainer = new BSPTaskLauncher(id, allocatedContainer, cm, conf, jobFile,
                jobId);

        launchers.put(id, runnableLaunchContainer);
        runnableLaunchContainer.start();
        completionQueue.add(runnableLaunchContainer);
        id++;
    }

    LOG.info("Waiting for tasks to finish...");
    state = JobState.RUNNING;
    int completed = 0;

    List<Integer> cleanupTasks = new ArrayList<Integer>();
    while (completed != numBSPTasks) {
        for (BSPTaskLauncher task : completionQueue) {
            BSPTaskStatus returnedTask = task.poll();
            // if our task returned with a finished state
            if (returnedTask != null) {
                if (returnedTask.getExitStatus() != 0) {
                    LOG.error("Task with id \"" + returnedTask.getId() + "\" failed!");
                    cleanupTask(returnedTask.getId());
                    state = JobState.FAILED;
                    return state;
                } else {
                    LOG.info("Task \"" + returnedTask.getId() + "\" sucessfully finished!");
                    completed++;
                    LOG.info("Waiting for " + (numBSPTasks - completed) + " tasks to finish!");
                }
                cleanupTasks.add(returnedTask.getId());
            }
        }
        Thread.sleep(1000L);
    }

    for (Integer stopId : cleanupTasks) {
        cleanupTask(stopId);
    }

    state = JobState.SUCCESS;
    return state;
}

---