Example usage for org.apache.hadoop.mapred JobConf setCombinerClass

List of usage examples for org.apache.hadoop.mapred JobConf setCombinerClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setCombinerClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setCombinerClass.

Prototype

public void setCombinerClass(Class<? extends Reducer> theClass)

Source Link

Document

Set the user-defined combiner class used to combine map-outputs before being sent to the reducers.

Usage

From source file:pegasus.heigen.L1norm.java

License:Apache License

protected JobConf configL1norm(Path in_path, Path out_path) throws Exception {
    final JobConf conf = new JobConf(getConf(), L1norm.class);
    conf.setJobName("L1norm");

    conf.setMapperClass(L1norm.MapStage1.class);
    conf.setReducerClass(PegasusUtils.RedSumDouble.class);
    conf.setCombinerClass(PegasusUtils.RedSumDouble.class);

    FileInputFormat.setInputPaths(conf, in_path);
    FileOutputFormat.setOutputPath(conf, out_path);

    conf.setNumReduceTasks(1);/*from   w  ww.j  a  v a  2s.  c  om*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(DoubleWritable.class);

    return conf;
}

From source file:pegasus.heigen.L1normBlock.java

License:Apache License

protected JobConf configL1norm(Path in_path, Path out_path) throws Exception {
    final JobConf conf = new JobConf(getConf(), L1normBlock.class);
    conf.set("block_width", "" + block_width);
    conf.setJobName("L1normBlock");

    conf.setMapperClass(MapStage1.class);
    conf.setReducerClass(PegasusUtils.RedSumDouble.class);
    conf.setCombinerClass(PegasusUtils.RedSumDouble.class);

    FileInputFormat.setInputPaths(conf, in_path);
    FileOutputFormat.setOutputPath(conf, out_path);

    conf.setNumReduceTasks(1);//from  w  ww.j  a  va 2s .  c  om

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(DoubleWritable.class);

    return conf;
}

From source file:pegasus.pagerank.PagerankBlock.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), PagerankNaive.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.set("mixing_c", "" + mixing_c);
    conf.set("converge_threshold", "" + converge_threshold);
    conf.setJobName("Pagerank_Stage3");

    conf.setMapperClass(PagerankNaive.MapStage3.class);
    conf.setReducerClass(PagerankNaive.RedStage3.class);
    conf.setCombinerClass(PagerankNaive.RedStage3.class);

    fs.delete(minmax_path, true);/*w  w  w. ja v a  2s.c om*/

    FileInputFormat.setInputPaths(conf, vector_unfold_path);
    FileOutputFormat.setOutputPath(conf, minmax_path);

    conf.setNumReduceTasks(1);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(DoubleWritable.class);

    return conf;
}

From source file:pegasus.pagerank.PagerankBlock.java

License:Apache License

protected JobConf configStage4(double min_pr, double max_pr) throws Exception {
    final JobConf conf = new JobConf(getConf(), PagerankNaive.class);
    conf.set("min_pr", "" + min_pr);
    conf.set("max_pr", "" + max_pr);

    conf.setJobName("Pagerank_Stage4");

    conf.setMapperClass(PagerankNaive.MapStage4.class);
    conf.setReducerClass(PagerankNaive.RedStage4.class);
    conf.setCombinerClass(PagerankNaive.RedStage4.class);

    fs.delete(distr_path, true);/*from w w  w.ja  v  a2  s . com*/

    FileInputFormat.setInputPaths(conf, vector_unfold_path);
    FileOutputFormat.setOutputPath(conf, distr_path);

    conf.setNumReduceTasks(nreducers);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.pagerank.PagerankNaive.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), PagerankNaive.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.set("mixing_c", "" + mixing_c);
    conf.set("converge_threshold", "" + converge_threshold);
    conf.setJobName("Pagerank_Stage3");

    conf.setMapperClass(MapStage3.class);
    conf.setReducerClass(RedStage3.class);
    conf.setCombinerClass(RedStage3.class);

    FileInputFormat.setInputPaths(conf, vector_path);
    FileOutputFormat.setOutputPath(conf, minmax_path);

    conf.setNumReduceTasks(1);/*  ww w. j  ava 2s  .  c o  m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(DoubleWritable.class);

    return conf;
}

From source file:pegasus.pagerank.PagerankNaive.java

License:Apache License

protected JobConf configStage4(double min_pr, double max_pr) throws Exception {
    final JobConf conf = new JobConf(getConf(), PagerankNaive.class);
    conf.set("min_pr", "" + min_pr);
    conf.set("max_pr", "" + max_pr);

    conf.setJobName("Pagerank_Stage4");

    conf.setMapperClass(MapStage4.class);
    conf.setReducerClass(RedStage4.class);
    conf.setCombinerClass(RedStage4.class);

    FileInputFormat.setInputPaths(conf, vector_path);
    FileOutputFormat.setOutputPath(conf, distr_path);

    conf.setNumReduceTasks(nreducers);/*from w w  w.  ja  v  a 2  s  . c  o m*/

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.rwr.RWRBlock.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), PagerankNaive.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.set("mixing_c", "" + mixing_c);
    conf.setJobName("RWRBlock_Stage3");

    conf.setMapperClass(PagerankNaive.MapStage3.class);
    conf.setReducerClass(PagerankNaive.RedStage3.class);
    conf.setCombinerClass(PagerankNaive.RedStage3.class);

    fs.delete(minmax_path, true);/*from  ww w  .  jav  a 2 s .c  o m*/

    FileInputFormat.setInputPaths(conf, vector_unfold_path);
    FileOutputFormat.setOutputPath(conf, minmax_path);

    conf.setNumReduceTasks(1);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(DoubleWritable.class);

    return conf;
}

From source file:pegasus.rwr.RWRBlock.java

License:Apache License

protected JobConf configStage4(double min_pr, double max_pr) throws Exception {
    final JobConf conf = new JobConf(getConf(), PagerankNaive.class);
    conf.set("min_pr", "" + min_pr);
    conf.set("max_pr", "" + max_pr);

    conf.setJobName("RWRBlock_Stage4");

    conf.setMapperClass(PagerankNaive.MapStage4.class);
    conf.setReducerClass(PagerankNaive.RedStage4.class);
    conf.setCombinerClass(PagerankNaive.RedStage4.class);

    fs.delete(distr_path, true);// w  w  w. j  a v  a2  s. com

    FileInputFormat.setInputPaths(conf, vector_unfold_path);
    FileOutputFormat.setOutputPath(conf, distr_path);

    conf.setNumReduceTasks(nreducers);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}

From source file:pegasus.rwr.RWRNaive.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), RWRNaive.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.set("mixing_c", "" + mixing_c);
    conf.setJobName("RWR_Stage3");

    conf.setMapperClass(MapStage3.class);
    conf.setReducerClass(RedStage3.class);
    conf.setCombinerClass(RedStage3.class);

    fs.delete(minmax_path, true);// w ww .  j  a  v  a2s  . co m

    FileInputFormat.setInputPaths(conf, vector_path);
    FileOutputFormat.setOutputPath(conf, minmax_path);

    conf.setNumReduceTasks(1);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(DoubleWritable.class);

    return conf;
}

From source file:pegasus.rwr.RWRNaive.java

License:Apache License

protected JobConf configStage4(double min_rwr, double max_rwr) throws Exception {
    final JobConf conf = new JobConf(getConf(), RWRNaive.class);
    conf.set("min_rwr", "" + min_rwr);
    conf.set("max_rwr", "" + max_rwr);

    conf.setJobName("RWR_Stage4");

    conf.setMapperClass(MapStage4.class);
    conf.setReducerClass(RedStage4.class);
    conf.setCombinerClass(RedStage4.class);

    fs.delete(distr_path, true);//from  w  w  w.  j a  v a  2s .co  m

    FileInputFormat.setInputPaths(conf, vector_path);
    FileOutputFormat.setOutputPath(conf, distr_path);

    conf.setNumReduceTasks(nreducers);

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}