Example usage for org.apache.hadoop.yarn.util Apps addToEnvironment

List of usage examples for org.apache.hadoop.yarn.util Apps addToEnvironment

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.util.*
  5. Apps
  6. addToEnvironment

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.util Apps addToEnvironment.

Prototype

@Deprecated
public static void addToEnvironment(Map<String, String> environment, String variable, String value)

Source Link

Document

This older version of this method is kept around for compatibility because downstream frameworks like Spark and Tez have been using it.

Usage

From source file:com.github.sakserv.minicluster.simpleyarnapp.Client.java

License:Apache License

@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
    }//  ww w  .j  a v a  2 s  . c  om
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}

From source file:com.github.sakserv.simpleyarnapp.Client.java

License:Apache License

private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
    }/*from   w w  w.ja  va 2 s . c o m*/
    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}

From source file:com.yahoo.storm.yarn.StormOnYarn.java

License:Open Source License

private void launchApp(String appName, String queue, int amMB, String storm_zip_location) throws Exception {
    LOG.debug("StormOnYarn:launchApp() ...");
    YarnClientApplication client_app = _yarn.createApplication();
    GetNewApplicationResponse app = client_app.getNewApplicationResponse();
    _appId = app.getApplicationId();/*from  w w  w .  j av  a 2  s  .  c  om*/
    LOG.debug("_appId:" + _appId);

    if (amMB > app.getMaximumResourceCapability().getMemory()) {
        //TODO need some sanity checks
        amMB = app.getMaximumResourceCapability().getMemory();
    }
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
    appContext.setApplicationId(app.getApplicationId());
    appContext.setApplicationName(appName);
    appContext.setQueue(queue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the
    // local resources
    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    // Copy the application master jar to the filesystem
    // Create a local resource to point to the destination jar path
    String appMasterJar = findContainingJar(MasterServer.class);
    FileSystem fs = FileSystem.get(_hadoopConf);
    Path src = new Path(appMasterJar);
    String appHome = Util.getApplicationHomeForId(_appId.toString());
    Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar");
    fs.copyFromLocalFile(false, true, src, dst);
    localResources.put("AppMaster.jar", Util.newYarnAppResource(fs, dst));

    String stormVersion = Util.getStormVersion();
    Path zip;
    if (storm_zip_location != null) {
        zip = new Path(storm_zip_location);
    } else {
        zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
    }
    _stormConf.put("storm.zip.path", zip.makeQualified(fs).toUri().getPath());
    LocalResourceVisibility visibility = LocalResourceVisibility.PUBLIC;
    _stormConf.put("storm.zip.visibility", "PUBLIC");
    if (!Util.isPublic(fs, zip)) {
        visibility = LocalResourceVisibility.APPLICATION;
        _stormConf.put("storm.zip.visibility", "APPLICATION");
    }
    localResources.put("storm", Util.newYarnAppResource(fs, zip, LocalResourceType.ARCHIVE, visibility));

    Path confDst = Util.createConfigurationFileInFs(fs, appHome, _stormConf, _hadoopConf);
    // establish a symbolic link to conf directory
    localResources.put("conf", Util.newYarnAppResource(fs, confDst));

    // Setup security tokens
    Path[] paths = new Path[3];
    paths[0] = dst;
    paths[1] = zip;
    paths[2] = confDst;
    Credentials credentials = new Credentials();
    TokenCache.obtainTokensForNamenodes(credentials, paths, _hadoopConf);
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    //security tokens for HDFS distributed cache
    amContainer.setTokens(securityTokens);

    // Set local resource info into app master container launch context
    amContainer.setLocalResources(localResources);

    // Set the env variables to be setup in the env where the application master
    // will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    // add the runtime classpath needed for tests to work
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./conf");
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar");

    //Make sure that AppMaster has access to all YARN JARs
    List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath");
    ProcessBuilder pb = new ProcessBuilder(yarn_classpath_cmd).redirectError(Redirect.INHERIT);
    LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]");
    pb.environment().putAll(System.getenv());
    Process proc = pb.start();
    BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream(), "UTF-8"));
    String line = "";
    String yarn_class_path = (String) _stormConf.get("storm.yarn.yarn_classpath");
    if (yarn_class_path == null) {
        StringBuilder yarn_class_path_builder = new StringBuilder();
        while ((line = reader.readLine()) != null) {
            yarn_class_path_builder.append(line);
        }
        yarn_class_path = yarn_class_path_builder.toString();
    }
    LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]");
    proc.waitFor();
    reader.close();
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path);

    String stormHomeInZip = Util.getStormHomeInZip(fs, zip, stormVersion);
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/*");
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./storm/" + stormHomeInZip + "/lib/*");

    String java_home = (String) _stormConf.get("storm.yarn.java_home");
    if (java_home == null)
        java_home = System.getenv("JAVA_HOME");

    if (java_home != null && !java_home.isEmpty())
        env.put("JAVA_HOME", java_home);
    LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]");

    env.put("appJar", appMasterJar);
    env.put("appName", appName);
    env.put("appId", new Integer(_appId.getId()).toString());
    env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    amContainer.setEnvironment(env);

    // Set the necessary command to execute the application master
    Vector<String> vargs = new Vector<String>();
    if (java_home != null && !java_home.isEmpty())
        vargs.add(env.get("JAVA_HOME") + "/bin/java");
    else
        vargs.add("java");
    vargs.add("-Dstorm.home=./storm/" + stormHomeInZip + "/");
    vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log");
    //vargs.add("-verbose:class");
    vargs.add("com.yahoo.storm.yarn.MasterServer");
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    // Set java executable command
    LOG.info("Setting up app master command:" + vargs);

    amContainer.setCommands(vargs);

    // Set up resource type requirements
    // For now, only memory is supported so we set memory requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amMB);
    appContext.setResource(capability);
    appContext.setAMContainerSpec(amContainer);

    _yarn.submitApplication(appContext);
}

From source file:com.yss.yarn.launch.APPLaunch.java

License:Open Source License

private void launchApp(String appName, String queue, int amMB, String appMasterJar, String classPath,
        String lanchMainClass, Map<String, String> runenv) throws Exception {

    YarnClientApplication client_app = _yarn.createApplication();
    GetNewApplicationResponse app = client_app.getNewApplicationResponse();

    _appId = app.getApplicationId();//w w w  .  j  a  v a  2s.c o  m
    LOG.debug("_appId:" + _appId);

    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    appContext.setApplicationId(app.getApplicationId());
    appContext.setApplicationName(appName);
    appContext.setQueue(queue);

    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LOG.info("Copy App Master jar from local filesystem and add to local environment");
    LOG.info("load  " + appMasterJar);

    FileSystem fs = FileSystem.get(_hadoopConf);
    Path src = new Path(appMasterJar);
    String appHome = YarnUtil.getApplicationHomeForId(_appId.toString());
    Path dst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + "AppMaster.jar");

    fs.copyFromLocalFile(false, true, src, dst);
    localResources.put("AppMaster.jar", YarnUtil.newYarnAppResource(fs, dst));

    amContainer.setLocalResources(localResources);

    LOG.info("Set the environment for the application master");

    Map<String, String> env = new HashMap<String, String>();

    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), "./AppMaster.jar");

    List<String> yarn_classpath_cmd = java.util.Arrays.asList("yarn", "classpath");

    LOG.info("YARN CLASSPATH COMMAND = [" + yarn_classpath_cmd + "]");

    String yarn_class_path = classPath;

    LOG.info("YARN CLASSPATH = [" + yarn_class_path + "]");
    Apps.addToEnvironment(env, Environment.CLASSPATH.name(), yarn_class_path);

    String java_home = System.getenv("JAVA_HOME");

    if ((java_home != null) && !java_home.isEmpty()) {
        env.put("JAVA_HOME", java_home);
    }

    LOG.info("Using JAVA_HOME = [" + env.get("JAVA_HOME") + "]");
    env.put("appJar", appMasterJar);
    env.put("appName", appName);
    env.put("appId", _appId.toString());
    env.put("STORM_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    env.putAll(runenv);
    amContainer.setEnvironment(env);

    // Set the necessary command to execute the application master
    Vector<String> vargs = new Vector<String>();

    if ((java_home != null) && !java_home.isEmpty()) {
        vargs.add(env.get("JAVA_HOME") + "/bin/java");
    } else {
        vargs.add("java");
    }

    vargs.add("-Dlogfile.name=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/master.log");

    vargs.add(lanchMainClass);
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Set java executable command
    LOG.info("Setting up app master command:" + vargs);
    amContainer.setCommands(vargs);

    Resource capability = Records.newRecord(Resource.class);

    appContext.setResource(capability);
    appContext.setAMContainerSpec(amContainer);
    appContext.setApplicationName(appName);
    _yarn.submitApplication(appContext);
}

From source file:gobblin.yarn.YarnHelixUtils.java

License:Apache License

/**
 * Get environment variables in a {@link java.util.Map} used when launching a Yarn container.
 *
 * @param yarnConfiguration a Hadoop {@link Configuration} object carrying Hadoop/Yarn configuration properties
 * @return a {@link java.util.Map} storing environment variables used when launching a Yarn container
 *//*  w w w. ja va 2s  . c  om*/
@SuppressWarnings("deprecation")
public static Map<String, String> getEnvironmentVariables(Configuration yarnConfiguration) {
    Map<String, String> environmentVariableMap = Maps.newHashMap();

    if (System.getenv().containsKey(ApplicationConstants.Environment.JAVA_HOME.key())) {
        Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.JAVA_HOME.key(),
                System.getenv(ApplicationConstants.Environment.JAVA_HOME.key()));
    }

    // Add jars/files in the working directory of the ApplicationMaster to the CLASSPATH
    Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
            ApplicationConstants.Environment.PWD.$());
    Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
            ApplicationConstants.Environment.PWD.$() + File.separator + "*");

    String[] classpaths = yarnConfiguration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
    if (classpaths != null) {
        for (String classpath : classpaths) {
            Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
                    classpath.trim());
        }
    }

    return environmentVariableMap;
}

From source file:hws.core.ContainerUtils.java

License:Apache License

public static void setupContainerEnv(Map<String, String> containerEnv, Configuration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(containerEnv, Environment.CLASSPATH.name(), c.trim());
    }//from w  w w.jav a2s. co  m
    Apps.addToEnvironment(containerEnv, Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}

From source file:MasteringYarn.DistributedShellClient.java

private void setupEnvironmentForApplicationMaster(Map<String, String> environmentMap) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(environmentMap, ApplicationConstants.Environment.CLASSPATH.name(), c.trim());
    }/*ww  w.jav a  2s .  com*/
    Apps.addToEnvironment(environmentMap, ApplicationConstants.Environment.CLASSPATH.name(),
            ApplicationConstants.Environment.PWD.$() + File.separator + "*");
}

From source file:runtime.starter.MPJAppMaster.java

License:Open Source License

private void setupEnv(Map<String, String> containerEnv) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {

        Apps.addToEnvironment(containerEnv, Environment.CLASSPATH.name(), c.trim());

    }//from w w w . j  a  v a  2s  .c  om

    Apps.addToEnvironment(containerEnv, Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}

From source file:runtime.starter.MPJYarnClient.java

License:Open Source License

private void setupAppMasterEnv(Map<String, String> appMasterEnv) {

    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
    }//from ww  w.ja  v a 2s .  c o  m

    Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}