Example usage for org.apache.hadoop.mapreduce MRJobConfig MAP_LOG_LEVEL

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce MRJobConfig MAP_LOG_LEVEL.

Prototype

String MAP_LOG_LEVEL

To view the source code for org.apache.hadoop.mapreduce MRJobConfig MAP_LOG_LEVEL.

Click Source Link

Usage

From source file:com.streamsets.pipeline.emr.EmrBinding.java

License:Apache License

@Override
public void init() throws Exception {
    Configuration conf = new Configuration();
    LOG.info("Arg 0: {}, Arg 1: {}, Arg 2: {}, Arg 3: {}, Arg 4: {}", args[0], args[1], args[2], args[3],
            args[4]);//from w  w w . j  a  v a 2s  .  c o  m
    try (InputStream in = getClass().getClassLoader().getResourceAsStream("cluster_sdc.properties")) {
        properties = new Properties();
        properties.load(in);
        String dataFormat = Utils.getHdfsDataFormat(properties);
        for (Object key : properties.keySet()) {
            String realKey = String.valueOf(key);
            // TODO - Override other configs set in HdfsSource
            if (overriddenConfs.contains(realKey)) {
                String value = Utils.getPropertyNotNull(properties, realKey);
                conf.set(realKey, value);
            }
        }
        String javaOpts = args[3];
        Integer mapMemoryMb = HadoopMapReduceBinding.getMapMemoryMb(javaOpts, conf);
        if (mapMemoryMb != null) {
            conf.set(HadoopMapReduceBinding.MAPREDUCE_MAP_MEMORY_MB, String.valueOf(mapMemoryMb));
        }
        conf.set(HadoopMapReduceBinding.MAPREDUCE_JAVA_OPTS, javaOpts);

        conf.setBoolean("mapreduce.map.speculative", false);
        conf.setBoolean("mapreduce.reduce.speculative", false);
        if ("AVRO".equalsIgnoreCase(dataFormat)) {
            conf.set(Job.INPUT_FORMAT_CLASS_ATTR, "org.apache.avro.mapreduce.AvroKeyInputFormat");
            conf.set(Job.MAP_OUTPUT_KEY_CLASS, "org.apache.avro.mapred.AvroKey");
        }

        conf.set(MRJobConfig.MAP_LOG_LEVEL, args[4]);
        job = Job.getInstance(conf, "StreamSets Data Collector: "
                + properties.getProperty(ClusterModeConstants.CLUSTER_PIPELINE_TITLE) + "::" + args[2]);
        for (String archive : Arrays.asList(args[0].split("\\s*,\\s*"))) {
            job.addCacheArchive(new URI(archive));
        }
        for (String libJar : Arrays.asList(args[1].split("\\s*,\\s*"))) {
            job.addFileToClassPath(new Path(libJar));
        }
        job.setJarByClass(this.getClass());
        job.setNumReduceTasks(0);
        if (!"AVRO".equalsIgnoreCase(dataFormat)) {
            job.setOutputKeyClass(NullWritable.class);
        }
        job.setMapperClass(PipelineMapper.class);

        job.setOutputValueClass(NullWritable.class);

        job.setOutputFormatClass(NullOutputFormat.class);
    }

}

---

From source file:org.apache.tez.mapreduce.hadoop.TestMRHelpers.java

License:Apache License

private Configuration createConfForJavaOptsTest() {
    Configuration conf = new Configuration(false);
    conf.set(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, "fooMapAdminOpts");
    conf.set(MRJobConfig.MAP_JAVA_OPTS, "fooMapJavaOpts");
    conf.set(MRJobConfig.MAP_LOG_LEVEL, "FATAL");
    conf.set(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, "fooReduceAdminOpts");
    conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "fooReduceJavaOpts");
    conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "TRACE");
    return conf;// w ww .  j ava2s  .c o m
}

---

From source file:org.apache.tez.mapreduce.hadoop.TestMRHelpers.java

License:Apache License

private Configuration setupConfigForMREnvTest() {
    JobConf conf = new JobConf(new Configuration());
    conf.set(MRJobConfig.MAP_ENV, "foo=map1,bar=map2");
    conf.set(MRJobConfig.REDUCE_ENV, "foo=red1,bar=red2");
    conf.set(MRJobConfig.MAP_LOG_LEVEL, "TRACE");
    conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "FATAL");
    final String mapredAdminUserEnv = Shell.WINDOWS
            ? "PATH=%PATH%" + File.pathSeparator + "%TEZ_ADMIN_ENV%\\bin"
            : "LD_LIBRARY_PATH=$TEZ_ADMIN_ENV_TEST/lib/native";

    conf.set(MRJobConfig.MAPRED_ADMIN_USER_ENV, mapredAdminUserEnv);
    return conf;//  w  w  w.  jav a2  s  .  com
}

---