Example usage for org.apache.hadoop.mapred JobConf setNumMapTasks

List of usage examples for org.apache.hadoop.mapred JobConf setNumMapTasks

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setNumMapTasks

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setNumMapTasks.

Prototype

public void setNumMapTasks(int n)

Source Link

Document

Set the number of map tasks for this job.

Usage

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CheckConvergence.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*from  w w  w .j  a  va  2 s . c  o m*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CheckConvergenceMapper.class);
    conf.setCombinerClass(CheckConvergenceReducer.class);
    conf.setReducerClass(CheckConvergenceReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(DoubleWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(DoubleWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CheckingData.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*  w  ww .  j a v  a2  s  .co m*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CheckingDataMapper.class);
    conf.setReducerClass(CheckingDataReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(CiteRank.NUM_REDUCE_TASKS);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CiteRankTool.java

License:Open Source License

/**
 * Create a configuration bound to this class, with various options set up
 *
 * @return the job/*from   w ww  . j  av  a 2 s .c o  m*/
 */
protected JobConf createConfiguration() {
    JobConf conf = new JobConf(getConf(), this.getClass());
    conf.setJobName(getName());
    conf.setJar(getJarName());
    conf.setNumMapTasks(NUM_MAP_TASKS);

    conf.setInt("dfs.replication", NUM_REPLICAS);
    return conf;
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CountPages.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }//from ww  w. j  a v  a2 s  .c o  m

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CountPagesMapper.class);
    conf.setCombinerClass(CountPagesReducer.class);
    conf.setReducerClass(CountPagesReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(LongWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(LongWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.DanglingPages.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*  www .j a  v a2  s  .  c  om*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(DanglingPagesMapper.class);
    conf.setCombinerClass(DanglingPagesReducer.class);
    conf.setReducerClass(DanglingPagesReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(DoubleWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(DoubleWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.InitializeRanks.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 3) {
        return usage("<input path> <output path> <number of pages>");
    }//from ww  w. ja  va 2  s  .c om

    JobConf conf = createInputOutputConfiguration(args);

    conf.set(CiteRankTool.RANK_COUNT, args[2]);

    conf.setMapperClass(InitializeRanksMapper.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(CiteRank.NUM_REDUCE_TASKS);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.UpdateRanks.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 4) {
        return usage("<input path> <output path> <number of pages> <dangling pages contribution>");
    }//from w  ww . jav a2 s .co  m

    JobConf conf = createInputOutputConfiguration(args);

    conf.set(CiteRankTool.RANK_COUNT, args[2]);
    conf.set(CiteRankTool.RANK_DANGLING, args[3]);

    conf.setMapperClass(UpdateRanksMapper.class);
    conf.setReducerClass(UpdateRanksReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(CiteRank.NUM_REDUCE_TASKS);

    return runJob(conf);
}

From source file:org.terrier.structures.indexing.singlepass.hadoop.TestSplitEmittedTerm.java

License:Mozilla Public License

/** single map, single reducer */
public void testSMSRCalculatePartition() throws Exception {
    final JobConf j = new JobConf();
    j.setNumMapTasks(1);
    final SETPartitioner p = new SETPartitioner();
    p.configure(j);//  w w w .ja v a  2 s  .c o m
    assertEquals(0, p.calculatePartition(0, 1));
}

From source file:org.terrier.structures.indexing.singlepass.hadoop.TestSplitEmittedTerm.java

License:Mozilla Public License

/** multiple map, single reducer */
public void testMMSRCalculatePartition() throws Exception {
    final JobConf j = new JobConf();
    final int maptasks = 20;
    j.setNumMapTasks(maptasks);
    final SETPartitioner p = new SETPartitioner();
    p.configure(j);/*from www . j  a  va 2 s.  c o  m*/
    assertEquals(0, p.calculatePartition(0, 1));
    assertEquals(0, p.calculatePartition(19, 1));
    assertEquals(0, p.calculatePartition(10, 1));

}

From source file:org.terrier.structures.indexing.singlepass.hadoop.TestSplitEmittedTerm.java

License:Mozilla Public License

/** multiple map, multiple reducer */
public void testMMMRCalculatePartition() throws Exception {
    final JobConf j = new JobConf();
    final int maptasks = 20;
    j.setNumMapTasks(maptasks);
    final SETPartitioner p = new SETPartitioner();
    p.configure(j);/*from   w  w  w  .j av a  2 s . c  o m*/

    assertEquals(0, p.calculatePartition(0, 2));
    assertEquals(0, p.calculatePartition(1, 2));
    assertEquals(0, p.calculatePartition(9, 2));
    assertEquals(1, p.calculatePartition(10, 2));
    assertEquals(1, p.calculatePartition(19, 2));
}