Example usage for org.apache.hadoop.yarn.api ContainerManagementProtocol startContainers

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api ContainerManagementProtocol startContainers.

Prototype

@Public
@Stable
StartContainersResponse startContainers(StartContainersRequest request) throws YarnException, IOException;

Source Link

Document

The ApplicationMaster provides a list of StartContainerRequest s to a NodeManager to start Container s allocated to it using this interface.

Usage

From source file:org.apache.hama.bsp.BSPTaskLauncher.java

License:Apache License

private GetContainerStatusesRequest setupContainer(Container allocatedContainer, ContainerManagementProtocol cm,
        String user, int id) throws IOException, YarnException {
    LOG.info("Setting up a container for user " + user + " with id of " + id + " and containerID of "
            + allocatedContainer.getId() + " as " + user);
    // Now we setup a ContainerLaunchContext
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

    // Set the local resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LocalResource packageResource = Records.newRecord(LocalResource.class);
    FileSystem fs = FileSystem.get(conf);
    Path packageFile = new Path(System.getenv(YARNBSPConstants.HAMA_YARN_LOCATION));
    URL packageUrl = ConverterUtils
            .getYarnUrlFromPath(packageFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()));
    LOG.info("PackageURL has been composed to " + packageUrl.toString());
    try {/*from w w  w. j  ava  2 s.c  om*/
        LOG.info("Reverting packageURL to path: " + ConverterUtils.getPathFromYarnURL(packageUrl));
    } catch (URISyntaxException e) {
        LOG.fatal("If you see this error the workarround does not work", e);
    }

    packageResource.setResource(packageUrl);
    packageResource.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_SIZE)));
    packageResource.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_TIMESTAMP)));
    packageResource.setType(LocalResourceType.FILE);
    packageResource.setVisibility(LocalResourceVisibility.APPLICATION);

    localResources.put(YARNBSPConstants.APP_MASTER_JAR_PATH, packageResource);

    Path hamaReleaseFile = new Path(System.getenv(YARNBSPConstants.HAMA_RELEASE_LOCATION));
    URL hamaReleaseUrl = ConverterUtils
            .getYarnUrlFromPath(hamaReleaseFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()));
    LOG.info("Hama release URL has been composed to " + hamaReleaseUrl.toString());

    LocalResource hamaReleaseRsrc = Records.newRecord(LocalResource.class);
    hamaReleaseRsrc.setResource(hamaReleaseUrl);
    hamaReleaseRsrc.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_SIZE)));
    hamaReleaseRsrc.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_TIMESTAMP)));
    hamaReleaseRsrc.setType(LocalResourceType.ARCHIVE);
    hamaReleaseRsrc.setVisibility(LocalResourceVisibility.APPLICATION);

    localResources.put(YARNBSPConstants.HAMA_SYMLINK, hamaReleaseRsrc);

    ctx.setLocalResources(localResources);

    /*
     * TODO Package classpath seems not to work if you're in pseudo distributed
     * mode, because the resource must not be moved, it will never be unpacked.
     * So we will check if our jar file has the file:// prefix and put it into
     * the CP directly
     */

    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$())
            .append(File.pathSeparatorChar).append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }

    classPathEnv.append(File.pathSeparator);
    classPathEnv
            .append("./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/*");
    classPathEnv.append(File.pathSeparator);
    classPathEnv.append(
            "./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/lib/*");

    Vector<CharSequence> vargs = new Vector<CharSequence>();
    vargs.add("${JAVA_HOME}/bin/java");
    vargs.add("-cp " + classPathEnv + "");
    vargs.add(BSPRunner.class.getCanonicalName());

    vargs.add(jobId.getJtIdentifier());
    vargs.add(Integer.toString(id));
    vargs.add(this.jobFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());

    ctx.setCommands(commands);
    LOG.info("Starting command: " + commands);

    StartContainerRequest startReq = Records.newRecord(StartContainerRequest.class);
    startReq.setContainerLaunchContext(ctx);
    startReq.setContainerToken(allocatedContainer.getContainerToken());

    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startReq);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    cm.startContainers(requestList);

    GetContainerStatusesRequest statusReq = Records.newRecord(GetContainerStatusesRequest.class);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(allocatedContainer.getId());
    statusReq.setContainerIds(containerIds);
    return statusReq;
}

---

From source file:org.apache.tajo.master.YarnContainerProxy.java

License:Apache License

@Override
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerLaunchContext commonContainerLaunchContext) {
    LOG.info("Launching Container with Id: " + containerID);
    if (this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
        state = ContainerState.DONE;//from  www . j  a va 2  s .c  o  m
        LOG.error("Container (" + containerID + " was killed before it was launched");
        return;
    }

    ContainerManagementProtocol proxy = null;
    try {

        proxy = getCMProxy(containerID, containerMgrAddress, containerToken);

        // Construct the actual Container
        ContainerLaunchContext containerLaunchContext = createContainerLaunchContext(
                commonContainerLaunchContext);

        // Now launch the actual container
        List<StartContainerRequest> startRequestList = new ArrayList<StartContainerRequest>();
        StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
        startRequest.setContainerLaunchContext(containerLaunchContext);
        startRequestList.add(startRequest);
        StartContainersRequest startRequests = Records.newRecord(StartContainersRequest.class);
        startRequests.setStartContainerRequests(startRequestList);
        StartContainersResponse response = proxy.startContainers(startRequests);

        ByteBuffer portInfo = response.getAllServicesMetaData().get(PullServerAuxService.PULLSERVER_SERVICEID);

        if (portInfo != null) {
            port = PullServerAuxService.deserializeMetaData(portInfo);
        }

        LOG.info("PullServer port returned by ContainerManager for " + containerID + " : " + port);

        if (port < 0) {
            this.state = ContainerState.FAILED;
            throw new IllegalStateException(
                    "Invalid shuffle port number " + port + " returned for " + containerID);
        }

        this.state = ContainerState.RUNNING;
        this.hostName = containerMgrAddress.split(":")[0];
        context.getResourceAllocator().addContainer(containerID, this);
    } catch (Throwable t) {
        String message = "Container launch failed for " + containerID + " : "
                + StringUtils.stringifyException(t);
        this.state = ContainerState.FAILED;
        LOG.error(message);
    } finally {
        if (proxy != null) {
            yarnRPC.stopProxy(proxy, conf);
        }
    }
}

---

From source file:org.springframework.yarn.am.AppmasterCmTemplate.java

License:Apache License

@Override
public StartContainersResponse startContainers(final StartContainersRequest request) {
    return execute(new YarnRpcCallback<StartContainersResponse, ContainerManagementProtocol>() {
        @Override/*w w  w. j a v a 2  s  .co  m*/
        public StartContainersResponse doInYarn(ContainerManagementProtocol proxy)
                throws YarnException, IOException {
            return proxy.startContainers(request);
        }
    });
}

---