Example usage for org.apache.hadoop.yarn.api.records ContainerLaunchContext setServiceData

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerLaunchContext setServiceData.

Prototype

@Public
@Stable
public abstract void setServiceData(Map<String, ByteBuffer> serviceData);

Source Link

Document

Set application-specific binary service data.

Usage

From source file:io.amient.yarn1.YarnContainerContext.java

License:Open Source License

public ContainerLaunchContext createContainerLaunchContext() throws IOException {
    ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class);
    context.setEnvironment(prepareEnvironment());
    context.setCommands(prepareCommands());
    context.setLocalResources(prepareLocalResources());
    Map<String, ByteBuffer> serviceData = Maps.newHashMap();
    context.setServiceData(serviceData);
    return context;
}

---

From source file:org.apache.tajo.master.TaskRunnerLauncherImpl.java

License:Apache License

private ContainerLaunchContext createCommonContainerLaunchContext() {
    TajoConf conf = (TajoConf) getConfig();

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    try {/* ww w  .  ja va 2 s . co m*/
        ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
        e.printStackTrace();
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the env variables to be setup
    ////////////////////////////////////////////////////////////////////////////
    LOG.info("Set the environment for the application master");

    Map<String, String> environment = new HashMap<String, String>();
    //String initialClassPath = getInitialClasspath(conf);
    environment.put(Environment.SHELL.name(), "/bin/bash");
    environment.put(Environment.JAVA_HOME.name(), System.getenv(Environment.JAVA_HOME.name()));

    // TODO - to be improved with org.apache.tajo.sh shell script
    Properties prop = System.getProperties();
    if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE")) {
        environment.put(Environment.CLASSPATH.name(), prop.getProperty("java.class.path", null));
    } else {
        // Add AppMaster.jar location to classpath
        // At some point we should not be required to add
        // the hadoop specific classpaths to the env.
        // It should be provided out of the box.
        // For now setting all required classpaths including
        // the classpath to "." for the application jar
        StringBuilder classPathEnv = new StringBuilder("./");
        //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
        for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
            classPathEnv.append(':');
            classPathEnv.append(c.trim());
        }

        classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH"));
        classPathEnv.append(":./log4j.properties:./*");
        environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME"));
        environment.put(Environment.HADOOP_COMMON_HOME.name(), System.getenv("HADOOP_HOME"));
        environment.put(Environment.HADOOP_HDFS_HOME.name(), System.getenv("HADOOP_HOME"));
        environment.put(Environment.HADOOP_YARN_HOME.name(), System.getenv("HADOOP_HOME"));
        environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH"));
        environment.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    }

    ctx.setEnvironment(environment);

    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    FileSystem fs = null;

    LOG.info("defaultFS: " + conf.get("fs.default.name"));
    LOG.info("defaultFS: " + conf.get("fs.defaultFS"));
    try {
        fs = FileSystem.get(conf);
    } catch (IOException e) {
        e.printStackTrace();
    }

    FileContext fsCtx = null;
    try {
        fsCtx = FileContext.getFileContext(getConfig());
    } catch (UnsupportedFileSystemException e) {
        e.printStackTrace();
    }

    LOG.info("Writing a QueryConf to HDFS and add to local environment");
    Path queryConfPath = new Path(fs.getHomeDirectory(), QueryConf.FILENAME);
    try {
        writeConf(conf, queryConfPath);

        LocalResource queryConfSrc = createApplicationResource(fsCtx, queryConfPath, LocalResourceType.FILE);
        localResources.put(QueryConf.FILENAME, queryConfSrc);

        ctx.setLocalResources(localResources);
    } catch (IOException e) {
        e.printStackTrace();
    }

    // Add shuffle token
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    try {
        //LOG.info("Putting shuffle token in serviceData");
        serviceData.put(PullServerAuxService.PULLSERVER_SERVICEID, PullServerAuxService.serializeMetaData(0));
    } catch (IOException ioe) {
        LOG.error(ioe);
    }
    ctx.setServiceData(serviceData);

    return ctx;
}

---

From source file:org.apache.tajo.master.YarnContainerProxy.java

License:Apache License

public static ContainerLaunchContext createCommonContainerLaunchContext(Configuration config, String queryId,
        boolean isMaster) {
    TajoConf conf = (TajoConf) config;/*  www .jav  a2  s .c o  m*/

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

    try {
        ByteBuffer userToken = ByteBuffer
                .wrap(UserGroupInformation.getCurrentUser().getShortUserName().getBytes());
        ctx.setTokens(userToken);
    } catch (IOException e) {
        e.printStackTrace();
    }

    ////////////////////////////////////////////////////////////////////////////
    // Set the env variables to be setup
    ////////////////////////////////////////////////////////////////////////////
    LOG.info("Set the environment for the application master");

    Map<String, String> environment = new HashMap<String, String>();
    //String initialClassPath = getInitialClasspath(conf);
    environment.put(ApplicationConstants.Environment.SHELL.name(), "/bin/bash");
    if (System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()) != null) {
        environment.put(ApplicationConstants.Environment.JAVA_HOME.name(),
                System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()));
    }

    // TODO - to be improved with org.apache.tajo.sh shell script
    Properties prop = System.getProperties();

    if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE")
            || (System.getenv("tajo.test") != null && System.getenv("tajo.test").equalsIgnoreCase("TRUE"))) {
        LOG.info("tajo.test is TRUE");
        environment.put(ApplicationConstants.Environment.CLASSPATH.name(),
                prop.getProperty("java.class.path", null));
        environment.put("tajo.test", "TRUE");
    } else {
        // Add AppMaster.jar location to classpath
        // At some point we should not be required to add
        // the hadoop specific classpaths to the env.
        // It should be provided out of the box.
        // For now setting all required classpaths including
        // the classpath to "." for the application jar
        StringBuilder classPathEnv = new StringBuilder("./");
        //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) {
        for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) {
            classPathEnv.append(':');
            classPathEnv.append(c.trim());
        }

        classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH"));
        classPathEnv.append(":./log4j.properties:./*");
        if (System.getenv("HADOOP_HOME") != null) {
            environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME"));
            environment.put(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name(),
                    System.getenv("HADOOP_HOME"));
            environment.put(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name(),
                    System.getenv("HADOOP_HOME"));
            environment.put(ApplicationConstants.Environment.HADOOP_YARN_HOME.name(),
                    System.getenv("HADOOP_HOME"));
        }

        if (System.getenv("TAJO_BASE_CLASSPATH") != null) {
            environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH"));
        }
        environment.put(ApplicationConstants.Environment.CLASSPATH.name(), classPathEnv.toString());
    }

    ctx.setEnvironment(environment);

    if (LOG.isDebugEnabled()) {
        LOG.debug("=================================================");
        for (Map.Entry<String, String> entry : environment.entrySet()) {
            LOG.debug(entry.getKey() + "=" + entry.getValue());
        }
        LOG.debug("=================================================");
    }
    ////////////////////////////////////////////////////////////////////////////
    // Set the local resources
    ////////////////////////////////////////////////////////////////////////////
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    LOG.info("defaultFS: " + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY));

    try {
        FileSystem fs = FileSystem.get(conf);
        FileContext fsCtx = FileContext.getFileContext(conf);
        Path systemConfPath = TajoConf.getSystemConfPath(conf);
        if (!fs.exists(systemConfPath)) {
            LOG.error("system_conf.xml (" + systemConfPath.toString() + ") Not Found");
        }
        LocalResource systemConfResource = createApplicationResource(fsCtx, systemConfPath,
                LocalResourceType.FILE);
        localResources.put(TajoConstants.SYSTEM_CONF_FILENAME, systemConfResource);
        ctx.setLocalResources(localResources);
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
    }

    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    try {
        serviceData.put(PullServerAuxService.PULLSERVER_SERVICEID, PullServerAuxService.serializeMetaData(0));
    } catch (IOException ioe) {
        LOG.error(ioe);
    }
    ctx.setServiceData(serviceData);

    return ctx;
}

---