Example usage for org.apache.hadoop.mapred JobConf setCombinerClass

List of usage examples for org.apache.hadoop.mapred JobConf setCombinerClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setCombinerClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setCombinerClass.

Prototype

public void setCombinerClass(Class<? extends Reducer> theClass)

Source Link

Document

Set the user-defined combiner class used to combine map-outputs before being sent to the reducers.

Usage

From source file:pegasus.ConCmptBlock.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), ConCmptBlock.class);
    conf.setJobName("ConCmptBlock_pass3");

    conf.setMapperClass(MapStage3.class);
    conf.setReducerClass(RedStage3.class);
    conf.setCombinerClass(RedStage3.class);

    FileInputFormat.setInputPaths(conf, nextbm_path);
    FileOutputFormat.setOutputPath(conf, output_path);

    conf.setNumReduceTasks(1);// This is necessary to summarize and save data.

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    return conf;/*from  ww w . j a  v  a 2  s .  c om*/
}

From source file:pegasus.ConCmptBlock.java

License:Apache License

protected JobConf configStage5() throws Exception {
    final JobConf conf = new JobConf(getConf(), ConCmptBlock.class);
    conf.set("block_width", "" + block_width);
    conf.setJobName("ConCmptBlock_pass5");

    conf.setMapperClass(MapStage5.class);
    conf.setReducerClass(RedStage5.class);
    conf.setCombinerClass(RedStage5.class);

    FileInputFormat.setInputPaths(conf, curbm_path);
    FileOutputFormat.setOutputPath(conf, summaryout_path);

    conf.setNumReduceTasks(nreducers);/*from   w  w  w  .  j  ava  2 s .co  m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.degdist.DegDist.java

License:Apache License

protected JobConf configPass1() throws Exception {
    final JobConf conf = new JobConf(getConf(), DegDist.class);
    conf.set("deg_type", "" + deg_type);

    conf.setJobName("DegDist_pass1");

    conf.setMapperClass(MapPass1.class);
    conf.setReducerClass(RedPass1.class);
    if (deg_type != InOutDeg) {
        conf.setCombinerClass(RedPass1.class);
    }/* w  ww.j a v  a  2  s .c  om*/

    FileInputFormat.setInputPaths(conf, edge_path);
    FileOutputFormat.setOutputPath(conf, node_deg_path);

    conf.setNumReduceTasks(nreducer);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.degdist.DegDist.java

License:Apache License

protected JobConf configPass2() throws Exception {
    final JobConf conf = new JobConf(getConf(), DegDist.class);

    conf.setJobName("DegDist_pass2");

    conf.setMapperClass(MapPass2.class);
    conf.setReducerClass(RedPass2.class);
    conf.setCombinerClass(RedPass2.class);

    FileInputFormat.setInputPaths(conf, node_deg_path);
    FileOutputFormat.setOutputPath(conf, deg_count_path);

    conf.setNumReduceTasks(nreducer);//from w  ww. j  av a 2s.  c o  m

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.hadi.Hadi.java

License:Apache License

protected JobConf configStage2() throws Exception {
    final JobConf conf = new JobConf(getConf(), Hadi.class);
    conf.set("nreplication", "" + nreplication);
    conf.set("encode_bitmask", "" + encode_bitmask);
    conf.set("cur_radius", "" + cur_radius);
    conf.setJobName("HADI_Stage2");

    conf.setMapperClass(MapStage2.class);
    conf.setReducerClass(RedStage2.class);
    conf.setCombinerClass(CombinerStage2.class);

    FileInputFormat.setInputPaths(conf, tempbm_path);
    FileOutputFormat.setOutputPath(conf, nextbm_path);

    conf.setNumReduceTasks(nreducer);/*from  ww  w  .j  a  v  a  2  s.c  o m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:pegasus.hadi.Hadi.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), Hadi.class);
    conf.set("nreplication", "" + nreplication);
    conf.set("encode_bitmask", "" + encode_bitmask);
    conf.setJobName("HADI_Stage3");

    conf.setMapperClass(MapStage3.class);
    conf.setReducerClass(RedStage3.class);
    conf.setCombinerClass(RedStage3.class);

    FileInputFormat.setInputPaths(conf, nextbm_path);
    FileOutputFormat.setOutputPath(conf, output_path);

    conf.setNumReduceTasks(nreducer);/*  w  w w  . ja v  a2s.c  om*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:pegasus.hadi.Hadi.java

License:Apache License

protected JobConf configStage5() throws Exception {
    final JobConf conf = new JobConf(getConf(), Hadi.class);
    conf.setJobName("HADI_Stage5");

    conf.setMapperClass(MapStage5.class);
    conf.setReducerClass(RedStage5.class);
    conf.setCombinerClass(RedStage5.class);

    FileInputFormat.setInputPaths(conf, radius_path);
    FileOutputFormat.setOutputPath(conf, radius_summary_path);

    conf.setNumReduceTasks(nreducer);/*from w  w w .  j av a2s . c  o  m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.hadi.HadiBlock.java

License:Apache License

protected JobConf configStage2() throws Exception {
    final JobConf conf = new JobConf(getConf(), HadiBlock.class);
    conf.set("nreplication", "" + nreplication);
    conf.set("encode_bitmask", "" + encode_bitmask);
    conf.set("cur_radius", "" + cur_radius);
    conf.set("block_width", "" + block_width);
    conf.setJobName("HADIBlk_Stage2");

    conf.setMapperClass(MapStage2.class);
    conf.setReducerClass(RedStage2.class);
    conf.setCombinerClass(CombinerStage2.class);

    FileInputFormat.setInputPaths(conf, tempbm_path);
    FileOutputFormat.setOutputPath(conf, nextbm_path);

    conf.setNumReduceTasks(nreducer);/*  w  w  w .j  a v  a 2 s  .co  m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:pegasus.hadi.HadiBlock.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), HadiBlock.class);
    conf.set("nreplication", "" + nreplication);
    conf.set("encode_bitmask", "" + encode_bitmask);
    conf.setJobName("HADIBlk_Stage3");

    conf.setMapperClass(MapStage3.class);
    conf.setReducerClass(RedStage3.class);
    conf.setCombinerClass(RedStage3.class);

    FileInputFormat.setInputPaths(conf, nextbm_path);
    FileOutputFormat.setOutputPath(conf, output_path);

    conf.setNumReduceTasks(nreducer);/*from   w  w w. java2  s .c  o m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:pegasus.hadi.HadiBlock.java

License:Apache License

protected JobConf configStage5() throws Exception {
    final JobConf conf = new JobConf(getConf(), HadiBlock.class);
    conf.setJobName("HADIBlk_Stage5");

    // reuse maper and reducers from Hadi class.
    conf.setMapperClass(Hadi.MapStage5.class);
    conf.setReducerClass(Hadi.RedStage5.class);
    conf.setCombinerClass(Hadi.RedStage5.class);

    FileInputFormat.setInputPaths(conf, radius_path);
    FileOutputFormat.setOutputPath(conf, radius_summary_path);

    conf.setNumReduceTasks(nreducer);//from w w w .j  av a 2s . c  o  m

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}