Example usage for org.apache.hadoop.yarn.client.api AMRMClient createAMRMClient

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api AMRMClient createAMRMClient.

Prototype

@Public
public static <T extends ContainerRequest> AMRMClient<T> createAMRMClient() 

Source Link

Document

Create a new instance of AMRMClient.

Usage

From source file:org.apache.sysml.yarn.DMLAppMaster.java

License:Apache License

public void runApplicationMaster(String[] args) throws YarnException, IOException {
    _conf = new YarnConfiguration();

    //obtain application ID
    String containerIdString = System.getenv(Environment.CONTAINER_ID.name());
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    _appId = containerId.getApplicationAttemptId().getApplicationId();
    LOG.info("SystemML appplication master (applicationID: " + _appId + ")");

    //initialize clients to ResourceManager
    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(_conf);//from  www . java2s. c o m
    rmClient.start();

    //register with ResourceManager
    rmClient.registerApplicationMaster("", 0, ""); //host, port for rm communication
    LOG.debug("Registered the SystemML application master with resource manager");

    //start status reporter to ResourceManager
    DMLAppMasterStatusReporter reporter = new DMLAppMasterStatusReporter(rmClient, 10000);
    reporter.start();
    LOG.debug("Started status reporter (heartbeat to resource manager)");

    //set DMLscript app master context
    DMLScript.setActiveAM();

    //parse input arguments
    String[] otherArgs = new GenericOptionsParser(_conf, args).getRemainingArgs();

    //run SystemML CP
    FinalApplicationStatus status = null;
    try {
        //core dml script execution (equivalent to non-AM runtime)
        boolean success = DMLScript.executeScript(_conf, otherArgs);

        if (success)
            status = FinalApplicationStatus.SUCCEEDED;
        else
            status = FinalApplicationStatus.FAILED;
    } catch (DMLScriptException ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script due to stop call:\n\t"
                + ex.getMessage());
        status = FinalApplicationStatus.FAILED;
        writeMessageToHDFSWorkingDir(ex.getMessage());
    } catch (Exception ex) {
        LOG.error(DMLYarnClient.APPMASTER_NAME + ": Failed to executed DML script.", ex);
        status = FinalApplicationStatus.FAILED;
    } finally {
        //stop periodic status reports
        reporter.stopStatusReporter();
        LOG.debug("Stopped status reporter");

        //unregister resource manager client
        rmClient.unregisterApplicationMaster(status, "", "");
        LOG.debug("Unregistered the SystemML application master");
    }
}

---

From source file:org.apache.twill.internal.yarn.Hadoop21YarnAMClient.java

License:Apache License

public Hadoop21YarnAMClient(Configuration conf) {
    super(ApplicationConstants.Environment.CONTAINER_ID.name());

    this.amrmClient = AMRMClient.createAMRMClient();
    this.amrmClient.init(conf);
    this.nmClient = new Hadoop21YarnNMClient(conf);
}

---

From source file:org.chenchun.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws IOException, YarnException, InterruptedException {
    final String params = args[0];
    final int containerNum = Integer.valueOf(args[1]);

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<AMRMClient.ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);//from www .  j a  v a 2s .c o  m
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    System.out.println("registerApplicationMaster 0");
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("registerApplicationMaster 1");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i = 0; i < containerNum; ++i) {
        AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null,
                priority);
        System.out.println("Making res-req " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    while (completedContainers < containerNum) {
        AllocateResponse response = rmClient.allocate(responseId++);
        System.out.println("Allocate response " + response.getAMCommand() + " " + "allocate "
                + response.getAllocatedContainers().size() + "contains");
        for (Container container : response.getAllocatedContainers()) {
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(params + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                            + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));
            System.out.println("Launching container " + container.getId() + " on " + container.getNodeId());
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            System.out.println("Completed container " + status.getContainerId());
        }
        Thread.sleep(1000);
    }
    System.out.println("Unregister ApplicationMaster");

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

---

From source file:org.elasticsearch.hadoop.yarn.am.AppMasterRpc.java

License:Apache License

public void start() {
    if (client != null) {
        return;/*from   www  . j  ava 2s  .c  o m*/
    }

    client = AMRMClient.createAMRMClient();
    YarnCompat.setNMTokenCache(client, nmTokenCache);
    client.init(cfg);
    client.start();
}

---

From source file:runtime.starter.MPJAppMaster.java

License:Open Source License

public void run() throws Exception {
    try {/*from  w  w  w. ja va 2 s  .  com*/
        appMasterSock = new Socket(serverName, ioServerPort);

        //redirecting stdout and stderr
        System.setOut(new PrintStream(appMasterSock.getOutputStream(), true));
        System.setErr(new PrintStream(appMasterSock.getOutputStream(), true));
    } catch (Exception exp) {
        exp.printStackTrace();
    }

    FileSystem fs = FileSystem.get(conf);
    Path wrapperDest = new Path(wrapperPath);
    FileStatus destStatus = fs.getFileStatus(wrapperDest);

    Path userFileDest = new Path(userJarPath);
    FileStatus destStatusClass = fs.getFileStatus(userFileDest);

    // Initialize AM <--> RM communication protocol
    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    // Initialize AM <--> NM communication protocol
    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    RegisterApplicationMasterResponse registerResponse = rmClient.registerApplicationMaster("", 0, "");
    // Priority for containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(mpjContainerPriority);

    maxMem = registerResponse.getMaximumResourceCapability().getMemory();

    if (debugYarn) {
        System.out.println("[MPJAppMaster]: Max memory capability resources " + "in cluster: " + maxMem);
    }

    if (containerMem > maxMem) {
        System.out.println("[MPJAppMaster]: container  memory specified above "
                + "threshold of cluster! Using maximum memory for " + "containers: " + containerMem);
        containerMem = maxMem;
    }

    maxCores = registerResponse.getMaximumResourceCapability().getVirtualCores();

    if (debugYarn) {
        System.out.println("[MPJAppMaster]: Max v-cores capability resources " + "in cluster: " + maxCores);
    }

    if (containerCores > maxCores) {
        System.out.println("[MPJAppMaster]: virtual cores specified above "
                + "threshold of cluster! Using maximum v-cores for " + "containers: " + containerCores);
        containerCores = maxCores;
    }

    // Resource requirements for containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMem);
    capability.setVirtualCores(containerCores);

    // Make container requests to ResourceManager
    for (int i = 0; i < np; ++i) {
        ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority);

        rmClient.addContainerRequest(containerReq);
    }

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    // Creating Local Resource for Wrapper   
    LocalResource wrapperJar = Records.newRecord(LocalResource.class);

    wrapperJar.setResource(ConverterUtils.getYarnUrlFromPath(wrapperDest));
    wrapperJar.setSize(destStatus.getLen());
    wrapperJar.setTimestamp(destStatus.getModificationTime());
    wrapperJar.setType(LocalResourceType.ARCHIVE);
    wrapperJar.setVisibility(LocalResourceVisibility.APPLICATION);

    // Creating Local Resource for UserClass
    LocalResource userClass = Records.newRecord(LocalResource.class);

    userClass.setResource(ConverterUtils.getYarnUrlFromPath(userFileDest));
    userClass.setSize(destStatusClass.getLen());
    userClass.setTimestamp(destStatusClass.getModificationTime());
    userClass.setType(LocalResourceType.ARCHIVE);
    userClass.setVisibility(LocalResourceVisibility.APPLICATION);

    localResources.put("mpj-yarn-wrapper.jar", wrapperJar);
    localResources.put("user-code.jar", userClass);

    while (allocatedContainers < np) {
        AllocateResponse response = rmClient.allocate(0);
        mpiContainers.addAll(response.getAllocatedContainers());
        allocatedContainers = mpiContainers.size();

        if (allocatedContainers != np) {
            Thread.sleep(100);
        }
    }

    if (debugYarn) {
        System.out.println("[MPJAppMaster]: launching " + allocatedContainers + " containers");
    }

    for (Container container : mpiContainers) {

        ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

        List<String> commands = new ArrayList<String>();

        commands.add(" $JAVA_HOME/bin/java");
        commands.add(" -Xmx" + containerMem + "m");
        commands.add(" runtime.starter.MPJYarnWrapper");
        commands.add("--serverName");
        commands.add(serverName); // server name
        commands.add("--ioServerPort");
        commands.add(Integer.toString(ioServerPort)); // IO server port
        commands.add("--deviceName");
        commands.add(deviceName); // device name
        commands.add("--className");
        commands.add(className); // class name
        commands.add("--psl");
        commands.add(psl); // protocol switch limit
        commands.add("--np");
        commands.add(Integer.toString(np)); // no. of containers
        commands.add("--rank");
        commands.add(" " + Integer.toString(rank++)); // rank

        //temp sock port to share rank and ports
        commands.add("--wireUpPort");
        commands.add(wireUpPort);

        if (appArgs != null) {
            commands.add("--appArgs");
            for (int i = 0; i < appArgs.length; i++) {
                commands.add(appArgs[i]);
            }
        }

        ctx.setCommands(commands);

        // Set local resource for containers
        ctx.setLocalResources(localResources);

        // Set environment for container
        Map<String, String> containerEnv = new HashMap<String, String>();
        setupEnv(containerEnv);
        ctx.setEnvironment(containerEnv);

        // Time to start the container
        nmClient.startContainer(container, ctx);

    }

    while (completedContainers < np) {
        // argument to allocate() is the progress indicator
        AllocateResponse response = rmClient.allocate(completedContainers / np);

        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            if (debugYarn) {
                System.out.println("\n[MPJAppMaster]: Container Id - " + status.getContainerId());
                System.out.println("[MPJAppMaster]: Container State - " + status.getState().toString());
                System.out.println("[MPJAppMaster]: Container Diagnostics - " + status.getDiagnostics());

            }
            ++completedContainers;
        }

        if (completedContainers != np) {
            Thread.sleep(100);
        }
        ;
    }
    // Un-register with ResourceManager 
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
    //shutDown AppMaster IO
    System.out.println("EXIT");
}

---

From source file:yrun.YarnRunnerApplicationMaster.java

License:Apache License

public void run(String[] args) throws Exception {
    JsonParser parser = new JsonParser();
    JsonElement element = parser.parse(new FileReader(YarnRunner.MASTER_JSON));

    LOG.info("Json [" + element + "]");

    JsonObject jsonObject = (JsonObject) element;
    int priority = jsonObject.get("priority").getAsInt();
    int numberOfContainers = jsonObject.get("numberOfContainers").getAsInt();
    int memory = jsonObject.get("memory").getAsInt();
    int vCores = jsonObject.get("vCores").getAsInt();
    String command = jsonObject.get("command").getAsString();

    // startHttpServer();
    // InetSocketAddress address = _server.getAddress();
    // LOG.info("Http server started at [" + address + "]");

    // String appHostName = "app-host-name";
    // int appHostPort = address.getPort();
    // String appTrackingUrl = "http://" + address.getHostName() + ":" +
    // appHostPort + "/";
    // LOG.info("App Tracking Url [" + appTrackingUrl + "]");

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);/*  w  w  w  .  j a  v a2  s .  com*/
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    LOG.info("Register Application Master 0");
    String appHostName = "";
    int appHostPort = 0;
    String appTrackingUrl = "";
    rmClient.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
    LOG.info("Register Application Master 1");

    // Priority for worker containers - priorities are intra-application
    Priority priorityRecord = Records.newRecord(Priority.class);
    priorityRecord.setPriority(priority);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    capability.setVirtualCores(vCores);

    // Make container requests to ResourceManager
    for (int i = 0; i < numberOfContainers; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priorityRecord);
        LOG.info("Making resource request for [" + i + "]");
        rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    long lastReport = 0;

    List<Container> containers = new ArrayList<Container>();

    while (completedContainers < numberOfContainers) {
        if (lastReport + TimeUnit.SECONDS.toNanos(5) < System.nanoTime()) {
            for (Container container : containers) {
                ContainerId containerId = container.getId();
                NodeId nodeId = container.getNodeId();
                ContainerStatus containerStatus = nmClient.getContainerStatus(containerId, nodeId);
                LOG.info("NodeId [" + nodeId + "] Container Status [" + containerStatus + "]");

                // Figure out

            }
            lastReport = System.nanoTime();
        }

        AllocateResponse response = rmClient.allocate(responseId++);
        for (Container container : response.getAllocatedContainers()) {
            containers.add(container);
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(command + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                            + "/stdout2" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));
            LOG.info("Launching container " + container.getId());
            nmClient.startContainer(container, ctx);
        }

        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            completedContainers++;
            LOG.info("Completed container " + status.getContainerId());
        }
        Thread.sleep(100);
    }
    // _server.stop(0);
    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

---

• «
• 1
• 2
• »