Example usage for org.apache.hadoop.mapreduce Job setCombinerClass

List of usage examples for org.apache.hadoop.mapreduce Job setCombinerClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setCombinerClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setCombinerClass.

Prototype

public void setCombinerClass(Class<? extends Reducer> cls) throws IllegalStateException

Source Link

Document

Set the combiner class for the job.

Usage

From source file:nl.utwente.bigdata.GoldenBoot.java

public static void main(String[] args) throws Exception {
    List<String> other_args = new ArrayList<String>();
    for (int i = 0; i < args.length; ++i) {
        other_args.add(args[i]);/*from  w ww  .  j av  a2 s  .  co m*/
    }
    Configuration conf = new Configuration();

    //First Job - GoalDefiner, the tweet that are containing goals within 5 minute blocks 
    Job job1 = new Job(conf, "GoalDefiner");
    job1.setJarByClass(GoalDefiner.class);
    job1.setMapperClass(GoalMapper.class);
    job1.setNumReduceTasks(0);
    job1.setOutputKeyClass(Text.class);
    job1.setOutputValueClass(Text.class);
    job1.setInputFormatClass(TextInputFormat.class);
    job1.setOutputFormatClass(TextOutputFormat.class);
    FileInputFormat.setInputPaths(job1, new Path(other_args.get(0)));
    FileOutputFormat.setOutputPath(job1, new Path("goal_definer_output"));

    job1.waitForCompletion(true);

    //Second Job - GoalScorerDefiner, Counts players within the goal tweets who scored probably the goal
    Job job2 = new Job(conf, "GoalScorerDefiner");
    job2.setJarByClass(GoalScorerDefiner.class);
    job2.setMapperClass(ScoreMapper.class);
    job2.setCombinerClass(ScoreReducer.class);
    job2.setReducerClass(ScoreReducer.class);
    job2.setOutputKeyClass(Text.class);
    job2.setOutputValueClass(Text.class);
    job2.setInputFormatClass(TextInputFormat.class);
    job2.setOutputFormatClass(TextOutputFormat.class);
    FileInputFormat.setInputPaths(job2, new Path("goal_definer_output/part*"));
    FileOutputFormat.setOutputPath(job2, new Path("goal_scorer_definer_output"));

    job2.waitForCompletion(true);

    //Third Job - GoalPlayerCount, counts all goals of a player
    Job job3 = new Job(conf, "GoalPlayerCount");
    job3.setJarByClass(GoalPlayerCount.class);
    job3.setMapperClass(CountMapper.class);
    job3.setCombinerClass(CountReducer.class);
    job3.setReducerClass(CountReducer.class);
    job3.setOutputKeyClass(Text.class);
    job3.setOutputValueClass(IntWritable.class);
    job3.setInputFormatClass(TextInputFormat.class);
    job3.setOutputFormatClass(TextOutputFormat.class);
    FileInputFormat.setInputPaths(job3, new Path("goal_scorer_definer_output/*"));
    FileOutputFormat.setOutputPath(job3, new Path("golden_boot_output"));

    job3.waitForCompletion(true);

}

From source file:nl.utwente.bigdata.MapReduce1.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: exampleTwitter <in> [<in>...] <out>");
        System.exit(2);/*from  w  w  w .ja v a  2  s. com*/
    }
    Job job = new Job(conf, "Twitter Reader");
    job.setJarByClass(MapReduce1.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MapReduce2.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: exampleTwitter <in> [<in>...] <out>");
        System.exit(2);/*ww  w  .j a va  2  s  .  co m*/
    }
    Job job = new Job(conf, "Twitter Reader");
    job.setJarByClass(MapReduce2.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setNumReduceTasks(1);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MapReduce2v1.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 3) {
        System.err.println("Usage: exampleTwitter <in>  <out> <name>");
        System.exit(2);//w  w w . j a  v a2 s.  c o m
    }
    Job job = new Job(conf, "Twitter Reader");
    job.getConfiguration().set("newLang", otherArgs[2]);
    job.setJarByClass(MapReduce2v1.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setNumReduceTasks(1);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MapReduce3.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: exampleTwitter <in> [<in>...] <out>");
        System.exit(2);// w w  w .j ava  2  s  .co  m
    }
    Job job = new Job(conf, "Twitter Reader");
    job.setJarByClass(MapReduce3.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MapReduce4.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: exampleTwitter <in> [<in>...] <out>");
        System.exit(2);//w  ww .java  2 s  .co m
    }
    Job job = new Job(conf, "Twitter Reader");
    job.setJarByClass(MapReduce4.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setNumReduceTasks(1);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MapReduce4v1.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 3) {
        System.err.println("Usage: exampleTwitter <in>  <out> <name>");
        System.exit(2);//  w w  w.  j  av a 2 s .  co  m
    }
    Job job = new Job(conf, "Twitter Reader");
    job.getConfiguration().set("newLang", otherArgs[2]);
    job.setJarByClass(MapReduce4v1.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setNumReduceTasks(1);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MapReduce5.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 8) {
        System.err.println("Usage: exampleTwitter <in>  <out> <year> <Month> <Day> <hour> <minute> <duration>");
        System.exit(2);//from  w w  w. java2s.c  o  m
    }
    Job job = new Job(conf, "Twitter Reader");
    job.getConfiguration().set("year", otherArgs[2]);
    job.getConfiguration().set("month", otherArgs[3]);
    job.getConfiguration().set("day", otherArgs[4]);
    job.getConfiguration().set("hour", otherArgs[5]);
    job.getConfiguration().set("minute", otherArgs[6]);
    job.getConfiguration().set("duration", otherArgs[7]);
    job.setJarByClass(MapReduce5.class);
    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);
    job.setCombinerClass(LanguageReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.MessageCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "count message per day");

    job.setJarByClass(MessageCount.class);
    job.setMapperClass(CountMapper2.class);
    job.setReducerClass(IntSumReducer2.class);
    job.setCombinerClass(IntSumReducer2.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:nl.utwente.bigdata.PlayersTweets.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {

        System.exit(2);/*from ww  w .j a  v a 2  s.c  om*/
    }
    Job job = new Job(conf, "Players Tweets");
    job.setJarByClass(PlayersTweets.class);
    job.setMapperClass(WorldCupMapper.class);
    job.setCombinerClass(WorldCupReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}