Example usage for org.apache.hadoop.yarn.api.records ContainerLaunchContext newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerLaunchContext newInstance.

Prototype

@Public
    @Unstable
    public static ContainerLaunchContext newInstance(Map<String, LocalResource> localResources,
            Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData,
            ByteBuffer tokens, Map<ApplicationAccessType, String> acls,
            ContainerRetryContext containerRetryContext) 

Source Link

Usage

From source file:com.github.hdl.tensorflow.yarn.app.LaunchContainerThread.java

License:Apache License

@Override
/**/*from   w  w w . j a v  a  2s  .  c om*/
 * Connects to CM, sets up container launch context
 * for shell command and eventually dispatches the container
 * start request to the CM.
 */
public void run() {
    LOG.info("Setting up container launch container for containerid=" + container.getId());

    FileSystem fs = null;
    try {
        fs = FileSystem.get(appMaster.getConfiguration());
    } catch (IOException e) {
        e.printStackTrace();
    }

    TFContainer tfContainer = new TFContainer(appMaster);

    Map<String, String> env = tfContainer.setJavaEnv(appMaster.getConfiguration(), null);
    tfContainer.setNativePath(env);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    ApplicationId appId = appMaster.getAppAttempId().getApplicationId();

    try {
        tfContainer.addToLocalResources(fs, tfServerJar, TFContainer.SERVER_JAR_PATH, localResources);
        if (jniSoDfsPath != null && !jniSoDfsPath.equals("")) {
            tfContainer.addToLocalResources(fs, jniSoDfsPath, "libbridge.so", localResources);
        }

    } catch (IOException e) {
        e.printStackTrace();
    }

    LOG.info("clusterspec: " + this.serverAddress.getClusterSpec().toString());
    //this.serverAddress.getClusterSpec().testClusterString();
    ClusterSpec cs = this.serverAddress.getClusterSpec();

    StringBuilder command = null;
    try {
        command = tfContainer.makeCommands(containerMemory, cs.getBase64EncodedJsonString(),
                this.serverAddress.getJobName(), this.serverAddress.getTaskIndex());
    } catch (JsonProcessingException e) {
        LOG.info("cluster spec cannot convert into base64 json string!");
        e.printStackTrace();
    } catch (ClusterSpecException e) {
        e.printStackTrace();
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    if (serverAddress != null) {
        LOG.info(serverAddress.getJobName() + " : " + serverAddress.getAddress() + ":"
                + serverAddress.getPort());
    }

    ContainerRetryContext containerRetryContext = ContainerRetryContext.newInstance(containerRetryPolicy,
            containerRetryErrorCodes, containerMaxRetries, containrRetryInterval);
    for (String cmd : commands) {
        LOG.info("Container " + container.getId() + " command: " + cmd.toString());
    }
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(localResources, env, commands, null,
            appMaster.getAllTokens().duplicate(), null, containerRetryContext);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
}

---

From source file:org.hdl.caffe.yarn.app.LaunchContainerThread.java

License:Apache License

@Override
/**/*from   w w  w  . java  2 s .  c  o m*/
 * Connects to CM, sets up container launch context
 * for shell command and eventually dispatches the container
 * start request to the CM.
 */
public void run() {
    LOG.info("Setting up container launch container for containerid=" + container.getId());

    FileSystem fs = null;
    try {
        fs = FileSystem.get(appMaster.getConfiguration());
    } catch (IOException e) {
        e.printStackTrace();
    }

    CaffeContainer caffeContainer = new CaffeContainer(appMaster);

    Map<String, String> env = caffeContainer.setJavaEnv(appMaster.getConfiguration(), null);
    caffeContainer.setNativePath(env);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    try {
        caffeContainer.addToLocalResources(fs, caffeProcessorJar, CaffeContainer.SERVER_JAR_PATH,
                localResources);
    } catch (IOException e) {
        e.printStackTrace();
    }

    LOG.info("cluster: " + this.serverAddress.getClusterSpec().toString());
    ClusterSpec cs = this.serverAddress.getClusterSpec();

    String command = null;
    try {
        command = caffeContainer.makeCommands(containerMemory, cs.getBase64EncodedJsonString(),
                this.serverAddress.getTaskIndex(), this.train, this.solver, this.feature, this.label,
                this.model, this.output, this.connection);
    } catch (JsonProcessingException e) {
        LOG.info("cluster spec cannot convert into base64 json string!");
        e.printStackTrace();
    } catch (ClusterSpecException e) {
        e.printStackTrace();
    }

    List<String> commands = new ArrayList<>();
    commands.add(command);
    if (serverAddress != null) {
        LOG.info(serverAddress.getAddress() + ":" + serverAddress.getPort());
    }

    ContainerRetryContext containerRetryContext = ContainerRetryContext.newInstance(containerRetryPolicy,
            containerRetryErrorCodes, containerMaxRetries, containrRetryInterval);
    for (String cmd : commands) {
        LOG.info("Container " + container.getId() + " command: " + cmd);
    }
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(localResources, env, commands, null,
            appMaster.getAllTokens().duplicate(), null, containerRetryContext);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
}

---

From source file:org.hdl.tensorflow.yarn.appmaster.LaunchContainerThread.java

License:Apache License

@Override
public void run() {
    try {/*  w w w. j  av a2  s .c o m*/
        Map<String, String> env = Utils.setJavaEnv(appMaster.getConfiguration());
        String current = ApplicationConstants.Environment.LD_LIBRARY_PATH.$$();
        env.put("LD_LIBRARY_PATH", current + ":" + "`pwd`");

        Map<String, Path> files = new HashMap<>();
        files.put(Constants.TF_JAR_NAME, new Path(tfJar));
        files.put(Constants.TF_LIB_NAME, new Path(tfLib));

        FileSystem fs = FileSystem.get(appMaster.getConfiguration());
        Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);

        String command = makeContainerCommand(containerMemory, clusterSpec.toBase64EncodedJsonString(),
                taskInfo.jobName, taskInfo.taskIndex);

        LOG.info("Launching a new container." + ", containerId=" + container.getId() + ", containerNode="
                + container.getNodeId().getHost() + ":" + container.getNodeId().getPort()
                + ", containerNodeURI=" + container.getNodeHttpAddress() + ", containerResourceMemory="
                + container.getResource().getMemorySize() + ", containerResourceVirtualCores="
                + container.getResource().getVirtualCores() + ", command: " + command);
        ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(localResources, env,
                Lists.newArrayList(command), null, null, null, null);
        appMaster.addContainer(container);
        appMaster.getNMClientAsync().startContainerAsync(container, ctx);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

---