Example usage for org.apache.hadoop.mapreduce MRJobConfig MAP_MEMORY

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce MRJobConfig MAP_MEMORY_MB.

Prototype

String MAP_MEMORY_MB

To view the source code for org.apache.hadoop.mapreduce MRJobConfig MAP_MEMORY_MB.

Click Source Link

Document

Configuration key for specifying memory requirement for the mapper.

Usage

From source file:org.apache.accumulo.test.mrit.IntegrationTestMapReduce.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    // read a list of tests from the input, and print out the results
    if (args.length != 2) {
        System.err.println("Wrong number of args: <input> <output>");
        return 1;
    }//w  ww.  ja  v  a 2  s .  com
    Configuration conf = getConf();
    Job job = Job.getInstance(conf, "accumulo integration test runner");
    conf = job.getConfiguration();

    // some tests take more than 10 minutes
    conf.setLong(MRJobConfig.TASK_TIMEOUT, 20 * 60 * 1000);

    // minicluster uses a lot of ram
    conf.setInt(MRJobConfig.MAP_MEMORY_MB, 4000);

    // hadoop puts an ancient version of jline on the classpath
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);

    // no need to run a test multiple times
    job.setSpeculativeExecution(false);

    // read one line at a time
    job.setInputFormatClass(NLineInputFormat.class);
    NLineInputFormat.setNumLinesPerSplit(job, 1);

    // run the test
    job.setJarByClass(IntegrationTestMapReduce.class);
    job.setMapperClass(TestMapper.class);

    // group test by result code
    job.setReducerClass(TestReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:org.apache.phoenix.util.PhoenixMRJobUtil.java

License:Apache License

/**
 * This method set the configuration values for Capacity scheduler.
 * @param conf - Configuration to which Capacity Queue information to be added
 *//*from   ww w  . j  ava  2 s  .  c  o m*/
public static void updateCapacityQueueInfo(Configuration conf) {
    conf.set(MRJobConfig.QUEUE_NAME, conf.get(PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY, DEFAULT_QUEUE_NAME));
    int mapMemoryMB = conf.getInt(PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY, DEFAULT_MAP_MEMROY_MB);

    conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemoryMB);
    conf.set(MRJobConfig.MAP_JAVA_OPTS, XMX_OPT + ((int) (mapMemoryMB * 0.9)) + "m");

    LOG.info("Queue Name=" + conf.get(MRJobConfig.QUEUE_NAME) + ";" + "Map Meory MB="
            + conf.get(MRJobConfig.MAP_MEMORY_MB) + ";" + "Map Java Opts="
            + conf.get(MRJobConfig.MAP_JAVA_OPTS));
}

From source file:org.apache.tez.mapreduce.hadoop.TestMRHelpers.java

License:Apache License

@Test(timeout = 5000)
public void testContainerResourceConstruction() {
    JobConf conf = new JobConf(new Configuration());
    Resource mapResource = MRHelpers.getResourceForMRMapper(conf);
    Resource reduceResource = MRHelpers.getResourceForMRReducer(conf);

    Assert.assertEquals(MRJobConfig.DEFAULT_MAP_CPU_VCORES, mapResource.getVirtualCores());
    Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB, mapResource.getMemory());
    Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_CPU_VCORES, reduceResource.getVirtualCores());
    Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB, reduceResource.getMemory());

    conf.setInt(MRJobConfig.MAP_CPU_VCORES, 2);
    conf.setInt(MRJobConfig.MAP_MEMORY_MB, 123);
    conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 20);
    conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 1234);

    mapResource = MRHelpers.getResourceForMRMapper(conf);
    reduceResource = MRHelpers.getResourceForMRReducer(conf);

    Assert.assertEquals(2, mapResource.getVirtualCores());
    Assert.assertEquals(123, mapResource.getMemory());
    Assert.assertEquals(20, reduceResource.getVirtualCores());
    Assert.assertEquals(1234, reduceResource.getMemory());
}

Example usage for org.apache.hadoop.mapreduce MRJobConfig MAP_MEMORY_MB

Introduction

Prototype

Document

Usage