Example usage for org.apache.hadoop.yarn.api.records ResourceRequest setNumContainers

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ResourceRequest setNumContainers.

Prototype

@Public
@Stable
public abstract void setNumContainers(int numContainers);

Source Link

Document

Set the number of containers required with the given specifications

Usage

From source file:com.cloudera.kitten.ContainerLaunchContextFactory.java

License:Open Source License

public ResourceRequest createResourceRequest(ContainerLaunchParameters parameters) {
    ResourceRequest req = Records.newRecord(ResourceRequest.class);
    req.setCapability(parameters.getContainerResource(clusterMin, clusterMax));
    req.setPriority(createPriority(parameters.getPriority()));
    req.setNumContainers(parameters.getNumInstances());
    req.setHostName("*"); // TODO: get smarter about this.
    return req;//www.  j av  a2 s  . c  o  m
}

---

From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

private void addResourceRequest(Priority priority, String resourceName, Resource capability,
        int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);
    if (remoteRequests == null) {
        remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
        this.remoteRequestsTable.put(priority, remoteRequests);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Added priority=" + priority);
        }/*from   www. j a  va2  s. c o m*/
    }
    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        reqMap = new HashMap<Resource, ResourceRequest>();
        remoteRequests.put(resourceName, reqMap);
    }
    ResourceRequest remoteRequest = reqMap.get(capability);
    if (remoteRequest == null) {
        remoteRequest = BuilderUtils.newResourceRequest(priority, resourceName, capability, 0);
        reqMap.put(capability, remoteRequest);
    }

    remoteRequest.setNumContainers(remoteRequest.getNumContainers() + containerCount);

    // Note this down for next interaction with ResourceManager
    addResourceRequestToAsk(remoteRequest);

    if (LOG.isDebugEnabled()) {
        LOG.debug("addResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

---

From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

private void decResourceRequest(Priority priority, String resourceName, Resource capability,
        int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);

    if (remoteRequests == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as priority " + priority + " is not present in request table");
        }//from   ww  w .  j a  v  a 2s. com
        return;
    }

    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as " + resourceName + " is not present in request table");
        }
        return;
    }
    ResourceRequest remoteRequest = reqMap.get(capability);

    if (LOG.isDebugEnabled()) {
        LOG.debug("BEFORE decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }

    remoteRequest.setNumContainers(remoteRequest.getNumContainers() - containerCount);
    if (remoteRequest.getNumContainers() < 0) {
        // guard against spurious removals
        remoteRequest.setNumContainers(0);
    }
    // send the ResourceRequest to RM even if is 0 because it needs to override
    // a previously sent value. If ResourceRequest was not sent previously then
    // sending 0 aught to be a no-op on RM
    addResourceRequestToAsk(remoteRequest);

    // delete entries from map if no longer needed
    if (remoteRequest.getNumContainers() == 0) {
        reqMap.remove(capability);
        if (reqMap.size() == 0) {
            remoteRequests.remove(resourceName);
        }
        if (remoteRequests.size() == 0) {
            remoteRequestsTable.remove(priority);
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.info("AFTER decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

---

From source file:com.datatorrent.stram.StramMiniClusterTest.java

License:Apache License

@Ignore
@Test/*from w  w  w.j  a  v  a  2  s.  c o  m*/
public void testUnmanagedAM() throws Exception {

    new InlineAM(conf) {
        @Override
        @SuppressWarnings("SleepWhileInLoop")
        public void runAM(ApplicationAttemptId attemptId) throws Exception {
            LOG.debug("AM running {}", attemptId);

            //AMRMClient amRmClient = new AMRMClientImpl(attemptId);
            //amRmClient.init(conf);
            //amRmClient.start();

            YarnClientHelper yarnClient = new YarnClientHelper(conf);
            ApplicationMasterProtocol resourceManager = yarnClient.connectToRM();

            // register with the RM (LAUNCHED -> RUNNING)
            RegisterApplicationMasterRequest appMasterRequest = Records
                    .newRecord(RegisterApplicationMasterRequest.class);
            resourceManager.registerApplicationMaster(appMasterRequest);

            // AM specific logic

            /*
            int containerCount = 1;
            Resource capability = Records.newRecord(Resource.class);
            capability.setMemory(1500);
                    
            Priority priority = Records.newRecord(Priority.class);
            priority.setPriority(10);
                    
            String[] hosts = {"vm1"};
            String[] racks = {"somerack"};
                    
            AMRMClient.ContainerRequest req = new AMRMClient.ContainerRequest(capability, hosts, racks, priority, containerCount);
            amRmClient.addContainerRequest(req);
                    
            for (int i=0; i<100; i++) {
              AllocateResponse ar = amRmClient.allocate(0);
              Thread.sleep(1000);
              LOG.debug("allocateResponse: {}" , ar);
            }
            */

            int responseId = 0;
            AllocateRequest req = Records.newRecord(AllocateRequest.class);
            req.setResponseId(responseId++);

            List<ResourceRequest> lr = Lists.newArrayList();
            lr.add(setupContainerAskForRM("hdev-vm", 1, 128, 10));
            lr.add(setupContainerAskForRM("/default-rack", 1, 128, 10));
            lr.add(setupContainerAskForRM("*", 1, 128, 10));

            req.setAskList(lr);

            LOG.info("Requesting: " + req.getAskList());
            resourceManager.allocate(req);

            for (int i = 0; i < 100; i++) {
                req = Records.newRecord(AllocateRequest.class);
                req.setResponseId(responseId++);

                AllocateResponse ar = resourceManager.allocate(req);
                sleep(1000);
                LOG.debug("allocateResponse: {}", ar);
            }

            // unregister from RM
            FinishApplicationMasterRequest finishReq = Records.newRecord(FinishApplicationMasterRequest.class);
            finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
            finishReq.setDiagnostics("testUnmanagedAM finished");
            resourceManager.finishApplicationMaster(finishReq);

        }

        private ResourceRequest setupContainerAskForRM(String resourceName, int numContainers,
                int containerMemory, int priority) {
            ResourceRequest request = Records.newRecord(ResourceRequest.class);

            // setup requirements for hosts
            // whether a particular rack/host is needed
            // Refer to apis under org.apache.hadoop.net for more
            // details on how to get figure out rack/host mapping.
            // using * as any host will do for the distributed shell app
            request.setResourceName(resourceName);

            // set no. of containers needed
            request.setNumContainers(numContainers);

            // set the priority for the request
            Priority pri = Records.newRecord(Priority.class);
            pri.setPriority(priority);
            request.setPriority(pri);

            // Set up resource type requirements
            // For now, only memory is supported so we set memory requirements
            Resource capability = Records.newRecord(Resource.class);
            capability.setMemory(containerMemory);
            request.setCapability(capability);

            return request;
        }

    }.run();

}

---

From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java

License:Apache License

private void setupAsk(AbstractProcess proc) {
    ContainerSpecification cSpec = proc.getContainerSpecification();
    ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);

    rsrcRequest.setHostName(cSpec.getHostname());

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);/*  ww w  . j  a  va 2s  . c o  m*/
    rsrcRequest.setPriority(pri);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(cSpec.getMemory());
    rsrcRequest.setCapability(capability);

    rsrcRequest.setNumContainers(1);

    AskRecord ar = new AskRecord();
    ar.req = rsrcRequest;
    ar.proc = proc;

    Set<AskRecord> arSet = resource2AskMap.get(capability);
    if (arSet == null) {
        arSet = new HashSet<AskRecord>();
        resource2AskMap.put(capability, arSet);
    }
    arSet.add(ar);
    proc2AskMap.put(proc, ar);

    System.err.println(proc + " -> [" + rsrcRequest.getHostName() + ", " + rsrcRequest.getNumContainers() + ", "
            + rsrcRequest.getPriority() + ", " + rsrcRequest.getCapability().getMemory() + "]");

    asks.add(rsrcRequest);
}

---

From source file:org.apache.hama.bsp.JobImpl.java

License:Apache License

private List<ResourceRequest> createBSPTaskRequest(int numTasks, int memoryInMb, int priority) {

    List<ResourceRequest> reqList = new ArrayList<ResourceRequest>(numTasks);
    for (int i = 0; i < numTasks; i++) {
        // Resource Request
        ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);

        // setup requirements for hosts
        // whether a particular rack/host is needed
        // useful for applications that are sensitive
        // to data locality
        rsrcRequest.setResourceName("*");

        // set the priority for the request
        Priority pri = Records.newRecord(Priority.class);
        pri.setPriority(priority);//from   w w w .ja  va  2s.  co m
        rsrcRequest.setPriority(pri);

        // Set up resource type requirements
        // For now, only memory is supported so we set memory requirements
        Resource capability = Records.newRecord(Resource.class);
        capability.setMemory(memoryInMb);
        rsrcRequest.setCapability(capability);

        // set no. of containers needed
        // matching the specifications
        rsrcRequest.setNumContainers(numBSPTasks);
        reqList.add(rsrcRequest);
    }
    return reqList;
}

---

From source file:org.apache.twill.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

private void decResourceRequest(Priority priority, String resourceName, Resource capability,
        int containerCount) {
    Map<String, Map<Resource, ResourceRequest>> remoteRequests = this.remoteRequestsTable.get(priority);

    if (remoteRequests == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as priority " + priority + " is not present in request table");
        }/*  www  . j a v a 2s .  c  om*/
        return;
    }

    Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
    if (reqMap == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not decrementing resource as " + resourceName + " is not present in request table");
        }
        return;
    }
    ResourceRequest remoteRequest = reqMap.get(capability);

    if (LOG.isDebugEnabled()) {
        LOG.debug("BEFORE decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }

    remoteRequest.setNumContainers(remoteRequest.getNumContainers() - containerCount);
    if (remoteRequest.getNumContainers() < 0) {
        // guard against spurious removals
        remoteRequest.setNumContainers(0);
    }
    // Send the ResourceRequest to RM even if is 0 because it needs to override
    // a previously sent value. If ResourceRequest was not sent previously then
    // sending 0 ought to be a no-op on RM.
    addResourceRequestToAsk(remoteRequest);

    // Delete entries from map if no longer needed.
    if (remoteRequest.getNumContainers() == 0) {
        reqMap.remove(capability);
        if (reqMap.size() == 0) {
            remoteRequests.remove(resourceName);
        }
        if (remoteRequests.size() == 0) {
            remoteRequestsTable.remove(priority);
        }
    }

    if (LOG.isDebugEnabled()) {
        LOG.info("AFTER decResourceRequest:" + " applicationId=" + appAttemptId + " priority="
                + priority.getPriority() + " resourceName=" + resourceName + " numContainers="
                + remoteRequest.getNumContainers() + " #asks=" + ask.size());
    }
}

---

From source file:org.deeplearning4j.iterativereduce.runtime.Utils.java

License:Apache License

public static ResourceRequest createResourceRequest(String host, int amount, int memory) {
    ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class);
    rsrcRequest.setHostName(host);//  www  .  j  a  va2 s. co  m

    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(0);
    rsrcRequest.setPriority(pri);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    rsrcRequest.setCapability(capability);

    rsrcRequest.setNumContainers(amount);

    LOG.debug("Created a resource request" + ", host=" + rsrcRequest.getHostName() + ", memory="
            + rsrcRequest.getCapability().getMemory() + ", amount=" + rsrcRequest.getNumContainers()
            + ", priority=" + rsrcRequest.getPriority().getPriority());

    return rsrcRequest;
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocator.java

License:Apache License

/**
 * Utility method creating a {@link ResourceRequest}.
 *
 * @param numContainers number of containers to request
 * @return request to be sent to resource manager
 *//*from  ww  w  . ja v  a2 s  . com*/
private ResourceRequest getContainerResourceRequest(int numContainers, String hostName) {
    ResourceRequest request = Records.newRecord(ResourceRequest.class);
    request.setResourceName(hostName);
    request.setNumContainers(numContainers);
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(priority);
    request.setPriority(pri);
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    ResourceCompat.setVirtualCores(capability, virtualcores);
    request.setCapability(capability);
    return request;
}

---