Example usage for org.apache.hadoop.yarn.client.api.async AMRMClientAsync addContainerRequest

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api.async AMRMClientAsync addContainerRequest.

Prototype

public abstract void addContainerRequest(T req);

Source Link

Document

Request containers for resources before calling allocate

Usage

From source file:com.gpiskas.yarn.AppMaster.java

License:Open Source License

public void run() throws Exception {
    conf = new YarnConfiguration();

    // Create NM Client
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);//from   ww w. jav a 2  s  . c om
    nmClient.start();

    // Create AM - RM Client
    AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
    rmClient.init(conf);
    rmClient.start();

    // Register with RM
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("AppMaster: Registered");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Reqiest Containers from RM
    System.out.println("AppMaster: Requesting " + containerCount + " Containers");
    for (int i = 0; i < containerCount; ++i) {
        rmClient.addContainerRequest(new ContainerRequest(capability, null, null, priority));
    }

    while (!containersFinished()) {
        Thread.sleep(100);
    }

    System.out.println("AppMaster: Unregistered");
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

---

From source file:hws.core.JobMaster.java

License:Apache License

public void runMainLoop() throws Exception {

    AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(100, this);
    rmClient.init(getConfiguration());//from   w  ww  .j ava  2 s .c  o  m
    rmClient.start();

    // Register with ResourceManager
    Logger.info("[AM] registerApplicationMaster 0");
    rmClient.registerApplicationMaster("", 0, "");
    Logger.info("[AM] registerApplicationMaster 1");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    final CountDownLatch doneLatch = new CountDownLatch(this.modulePipeline.size());
    // Make container requests to ResourceManager
    for (ModuleInfo moduleInfo : this.modulePipeline) { //create containers for each instance of each module
        zk.createPersistent("/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name(), "");
        zk.createPersistent(
                "/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name() + "/finish", "");
        zk.createPersistent(
                "/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name() + "/halted", "");
        zk.subscribeChildChanges(
                "/hadoop-watershed/" + this.appIdStr + "/" + moduleInfo.filterInfo().name() + "/finish",
                createFinishListener(moduleInfo.filterInfo().name(), moduleInfo.numFilterInstances(),
                        doneLatch));
        for (int i = 0; i < moduleInfo.numFilterInstances(); i++) {
            this.numContainersToWaitFor++;
            ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
            Logger.info("[AM] Making res-req for " + moduleInfo.filterInfo().name() + " " + i);
            rmClient.addContainerRequest(containerAsk);
        }
    }
    //TODO: process for starting the whole application
    //create containers
    // -> create instances
    // -> start output channels and filters
    // -> start input channels in reversed topological order (considering that there is no cycle)
    //    * if there is cycle, then inicially start in any order
    //TODO "send" the start signal via ZooKeeper

    Logger.info("[AM] waiting for containers to finish");
    try {
        doneLatch.await(); //await the input threads to finish
    } catch (InterruptedException e) {
        Logger.fatal(e.toString());
        //e.printStackTrace();
    }
    /*while(!doneWithContainers()) {
    Thread.sleep(50);
    }*/

    zk.createPersistent("/hadoop-watershed/" + appIdStr + "/done", "");

    Logger.info("[AM] unregisterApplicationMaster 0");
    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
    Logger.info("[AM] unregisterApplicationMaster 1");
}

---