Example usage for org.apache.hadoop.mapreduce.v2.util MRApps setClasspath

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.v2.util MRApps setClasspath.

Prototype

@SuppressWarnings("deprecation")
    public static void setClasspath(Map<String, String> environment, Configuration conf) throws IOException 

Source Link

Usage

From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java

License:Open Source License

/**
 * /*from   w  ww. j a v  a 2  s.  co m*/
 * @param yconf
 * @return
 * @throws IOException
 */
private Map<String, String> constructEnvionmentMap(YarnConfiguration yconf) throws IOException {
    Map<String, String> eMap = new HashMap<String, String>();

    //setup default app master environment
    StringBuilder classpath = new StringBuilder();
    for (String value : yconf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(value.trim());
    }

    //setup mapreduce appmaster environment (for robustness if not included in default environment)
    //for example, by default HDP 2.2 did not include mapred client libraries in this configuration
    //note: we cannot use mapreduce.application.classpath because it refers to HDFS and $PWD that needs to be setup       
    Map<String, String> env = System.getenv();
    String mapred_home = null;
    //get mapred home via alternative environment variables
    if (env.containsKey(MAPRED_HOME_ENV_CONST)) {
        mapred_home = env.get(MAPRED_HOME_ENV_CONST);
    } else if (env.containsKey(HADOOP_HOME_ENV_CONST)) {
        String tmp = env.get(HADOOP_HOME_ENV_CONST);
        mapred_home = tmp + File.separator + ".." + File.separator + "hadoop-mapreduce";
    }
    //concatenate mapred home libs to classpath
    if (mapred_home != null) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "*");
        classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "lib" + File.separator + "*");
    }

    eMap.put(Environment.CLASSPATH.name(), classpath.toString());
    MRApps.setClasspath(eMap, yconf);

    LOG.debug("Constructed environment classpath: " + classpath.toString());

    return eMap;
}

---

From source file:org.apache.sysml.yarn.DMLYarnClient.java

License:Apache License

private static Map<String, String> constructEnvionmentMap(YarnConfiguration yconf) throws IOException {
    Map<String, String> eMap = new HashMap<>();

    //setup default app master environment
    StringBuilder classpath = new StringBuilder();
    for (String value : yconf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(value.trim());//from w w  w.j  av  a2s.  c om
    }

    //setup mapreduce appmaster environment (for robustness if not included in default environment)
    //for example, by default HDP 2.2 did not include mapred client libraries in this configuration
    //note: we cannot use MRConfigurationNames.MR_APPLICATION_CLASSPATH because it refers to HDFS and $PWD that needs to be setup
    Map<String, String> env = System.getenv();
    String mapred_home = null;
    //get mapred home via alternative environment variables
    if (env.containsKey(MAPRED_HOME_ENV_CONST)) {
        mapred_home = env.get(MAPRED_HOME_ENV_CONST);
    } else if (env.containsKey(HADOOP_HOME_ENV_CONST)) {
        String tmp = env.get(HADOOP_HOME_ENV_CONST);
        mapred_home = tmp + File.separator + ".." + File.separator + "hadoop-mapreduce";
    }
    //concatenate mapred home libs to classpath
    if (mapred_home != null) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "*");
        classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "lib" + File.separator + "*");
    }

    eMap.put(Environment.CLASSPATH.name(), classpath.toString());
    MRApps.setClasspath(eMap, yconf);

    LOG.debug("Constructed environment classpath: " + classpath.toString());

    return eMap;
}

---