Example usage for org.apache.hadoop.yarn.client.api AMRMClient updateBlacklist

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api AMRMClient updateBlacklist.

Prototype

public abstract void updateBlacklist(List<String> blacklistAdditions, List<String> blacklistRemovals);

Source Link

Document

Update application's blacklist with addition or removal resources.

Usage

From source file:com.datatorrent.stram.BlacklistBasedResourceRequestHandler.java

License:Apache License

@Override
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient,
        Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources,
        int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests,
        List<ContainerRequest> removedContainerRequests) {
    // Issue all host specific requests first
    if (!hostSpecificRequestsMap.isEmpty() && requestedResources.isEmpty()) {
        LOG.info("Issue Host specific requests first");
        // Blacklist all the nodes and issue request for host specific
        Entry<String, List<ContainerRequest>> set = hostSpecificRequestsMap.entrySet().iterator().next();
        List<ContainerRequest> requests = set.getValue();
        List<String> blacklistNodes = resourceRequestor.getNodesExceptHost(requests.get(0).getNodes());
        amRmClient.updateBlacklist(blacklistNodes, requests.get(0).getNodes());
        blacklistedNodesForHostSpecificRequests = blacklistNodes;
        LOG.info("Sending {} request(s) after blacklisting all nodes other than {}", requests.size(),
                requests.get(0).getNodes());

        for (ContainerRequest cr : requests) {
            ContainerStartRequest csr = hostSpecificRequests.get(cr);
            ContainerRequest newCr = new ContainerRequest(cr.getCapability(), null, null, cr.getPriority());
            MutablePair<Integer, ContainerRequest> pair = new MutablePair<Integer, ContainerRequest>(
                    loopCounter, newCr);
            requestedResources.put(csr, pair);
            containerRequests.add(newCr);
            hostSpecificRequests.remove(cr);
        }/*from  w  ww  . j a  va2  s .co  m*/
        hostSpecificRequestsMap.remove(set.getKey());
    } else if (!requestedResources.isEmpty()) {
        // Check if any requests timed out, create new requests in that case
        recreateContainerRequest(requestedResources, loopCounter, resourceRequestor, removedContainerRequests);
    } else {
        if (blacklistedNodesForHostSpecificRequests != null) {
            // Remove the blacklisted nodes during host specific requests
            LOG.debug("All requests done.. Removing nodes from blacklist {}",
                    blacklistedNodesForHostSpecificRequests);
            amRmClient.updateBlacklist(null, blacklistedNodesForHostSpecificRequests);
            blacklistedNodesForHostSpecificRequests = null;
        }
        // Proceed with other requests after host specific requests are done
        if (!otherContainerRequests.isEmpty()) {
            for (Entry<ContainerRequest, ContainerStartRequest> entry : otherContainerRequests.entrySet()) {
                ContainerRequest cr = entry.getKey();
                ContainerStartRequest csr = entry.getValue();
                MutablePair<Integer, ContainerRequest> pair = new MutablePair<Integer, ContainerRequest>(
                        loopCounter, cr);
                requestedResources.put(csr, pair);
                containerRequests.add(cr);
            }
            otherContainerRequests.clear();
        }
    }
}

---