Example usage for org.apache.hadoop.mapred JobConf MAPRED_MAP_TASK

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf MAPRED_MAP_TASK_ENV.

Prototype

String MAPRED_MAP_TASK_ENV

To view the source code for org.apache.hadoop.mapred JobConf MAPRED_MAP_TASK_ENV.

Click Source Link

Document

Configuration key to set the environment of the child map tasks.

Usage

From source file:com.google.mr4c.hadoop.HadoopUtils.java

License:Open Source License

/**
  * @param varMap apply environment variable values from this map
  * @param vars apply existing values of these environment variables
*//*from  w w  w  .j a  va  2s  .c  o  m*/
public static void applyEnvironmentVariables(JobConf conf, Map<String, String> varMap, List<String> vars) {
    Map<String, String> allMap = new HashMap<String, String>(System.getenv());
    allMap.keySet().retainAll(vars); // only the env we wanted
    allMap.putAll(varMap);
    List<String> assigns = new ArrayList<String>();
    for (String var : allMap.keySet()) {
        String val = allMap.get(var);
        if (!StringUtils.isEmpty(val)) {
            assigns.add(var + "=" + val);
        }
    }
    String value = StringUtils.join(assigns, ", ");
    conf.set(JobConf.MAPRED_MAP_TASK_ENV, value);
    conf.set(JobConf.MAPRED_REDUCE_TASK_ENV, value);
}

From source file:hadoop.UIUCWikifierAppHadoop.java

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    JobConf job = new JobConf(conf, UIUCWikifierAppHadoop.class);

    //      System.out.println("Run.. Envinronment Variables");
    //      java.util.Map<String,String> env = System.getenv();
    ////from   w ww  . j  a v a 2s.  c  o  m
    //      System.out.println("Printing environment variables");
    //      for(String k : env.keySet()){
    //         System.out.println(k + "\t" + env.get(k));
    //      }
    //      String jlpValue = System.getProperty("java.library.path");
    //      System.out.println("java.library.path=" + jlpValue);
    //      System.setProperty("java.library.path", jlpValue + ":" + "/home/jgilme1/bin/gurobi550/linux64/lib");

    //process command line options
    Path in = new Path(args[0]);
    Path out = new Path(args[1]);

    //change current working directory to hdfs path..
    job.setJobName("entitylinker");
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setInputFormat(DistributeInputFormat.class);
    job.setOutputFormat(TextOutputFormat.class);
    job.setMapperClass(Map.class);
    FileInputFormat.setInputPaths(job, in);
    FileOutputFormat.setOutputPath(job, out);
    job.setNumReduceTasks(0);
    job.setNumMapTasks(Integer.parseInt(args[2]));
    job.set("mapreduce.input.fileinputformat.split.minsize", "0");
    job.set("mapred.child.java.opts", "-Xmx16g");
    job.setNumTasksToExecutePerJvm(-1);
    //job.setMemoryForMapTask(new Long(12288));
    //job.set(JobConf.MAPRED_MAP_TASK_ULIMIT, "12582912");

    String gurobiHomeVariable = "GUROBI_HOME";
    String gurobiHomeValue = "/home/jgilme1/bin/gurobi560/linux64";
    String pathVariable = "PATH";
    String newPathValue = gurobiHomeValue + "/bin";
    String ldLibraryPathVariable = "LD_LIBRARY_PATH";
    String ldLibraryPathValue = gurobiHomeValue + "/lib";
    String grbLicenseFileVariable = "GRB_LICENSE_FILE";
    String grbLicenseFileValue = "/scratch6/usr/jgilme1/gurobiLicense/gurobi.lic";

    StringBuilder newEnvironment = new StringBuilder();
    newEnvironment.append(gurobiHomeVariable);
    newEnvironment.append("=");
    newEnvironment.append(gurobiHomeValue);
    newEnvironment.append(",");
    newEnvironment.append(pathVariable);
    newEnvironment.append("=");
    newEnvironment.append("$" + pathVariable + ":");
    newEnvironment.append(newPathValue);
    newEnvironment.append(",");
    newEnvironment.append(ldLibraryPathVariable);
    newEnvironment.append("=$" + ldLibraryPathVariable + ":");
    newEnvironment.append(ldLibraryPathValue);
    newEnvironment.append(",");
    newEnvironment.append(grbLicenseFileVariable);
    newEnvironment.append("=");
    newEnvironment.append(grbLicenseFileValue);

    //System.out.println(newEnvironment.toString());
    job.set(JobConf.MAPRED_MAP_TASK_ENV, newEnvironment.toString());

    DistributedCache.addCacheArchive(new URI("/user/jgilme1/entitylinking/Wikifier2013.tar.gz"), job);

    JobClient.runJob(job);
    return 0;
}

Example usage for org.apache.hadoop.mapred JobConf MAPRED_MAP_TASK_ENV

Introduction

Prototype

Document

Usage