Example usage for org.apache.hadoop.mapreduce Job setGroupingComparatorClass

List of usage examples for org.apache.hadoop.mapreduce Job setGroupingComparatorClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setGroupingComparatorClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setGroupingComparatorClass.

Prototype

public void setGroupingComparatorClass(Class<? extends RawComparator> cls) throws IllegalStateException

Source Link

Document

Define the comparator that controls which keys are grouped together for a single call to Reducer#reduce(Object,Iterable,org.apache.hadoop.mapreduce.Reducer.Context)

Usage

From source file:ph.fingra.hadoop.mapred.parts.performance.FrequencyStatistic.java

License:Apache License

public Job createHourJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("hour", targetdate.getHour());

    Job job = new Job(conf);
    String jobName = "perform/tokenfreq hour job";
    job.setJobName(jobName);/*from w  ww .j a  v a  2s  .com*/

    job.setJarByClass(FrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(TokenfreqHourMapper.class);
    job.setReducerClass(TokenfreqHourReducer.class);

    job.setMapOutputKeyClass(TokenfreqHourKey.class);
    job.setMapOutputValueClass(TokenfreqHourEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(TokenfreqHourPartitioner.class);
    job.setSortComparatorClass(TokenfreqHourSortComparator.class);
    job.setGroupingComparatorClass(TokenfreqHourGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.HourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/hoursession job";
    job.setJobName(jobName);//from ww w  . ja  v  a  2 s .c o m

    job.setJarByClass(HourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(HourSessionMapper.class);
    job.setReducerClass(HourSessionReducer.class);

    job.setMapOutputKeyClass(HourSessionKey.class);
    job.setMapOutputValueClass(HourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(HourSessionPartitioner.class);
    job.setSortComparatorClass(HourSessionSortComparator.class);
    job.setGroupingComparatorClass(HourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.SessionLengthStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/sesstime job";
    job.setJobName(jobName);/*  ww  w.jav  a  2  s .com*/

    job.setJarByClass(SessionLengthStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(SesstimeMapper.class);
    job.setReducerClass(SesstimeReducer.class);

    job.setMapOutputKeyClass(SesstimeKey.class);
    job.setMapOutputValueClass(SesstimeEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(SesstimePartitioner.class);
    job.setSortComparatorClass(SesstimeSortComparator.class);
    job.setGroupingComparatorClass(SesstimeGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.SessionLengthStatistic.java

License:Apache License

public Job createHourJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("hour", targetdate.getHour());

    Job job = new Job(conf);
    String jobName = "perform/sesstime hour job";
    job.setJobName(jobName);/*  w w  w  . j a  v  a2  s.  co m*/

    job.setJarByClass(SessionLengthStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(SesstimeHourMapper.class);
    job.setReducerClass(SesstimeHourReducer.class);

    job.setMapOutputKeyClass(SesstimeHourKey.class);
    job.setMapOutputValueClass(SesstimeHourEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(SesstimeHourPartitioner.class);
    job.setSortComparatorClass(SesstimeHourSortComparator.class);
    job.setGroupingComparatorClass(SesstimeHourGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.UserSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/usersession job";
    job.setJobName(jobName);//from  www  . j a v a2s  .  co m

    job.setJarByClass(UserSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(UserSessionMapper.class);
    job.setReducerClass(UserSessionReducer.class);

    job.setMapOutputKeyClass(UserSessionKey.class);
    job.setMapOutputValueClass(UserSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(UserSessionPartitioner.class);
    job.setSortComparatorClass(UserSessionSortComparator.class);
    job.setGroupingComparatorClass(UserSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.UserSessionStatistic.java

License:Apache License

public Job createHourJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("hour", targetdate.getHour());

    Job job = new Job(conf);
    String jobName = "perform/usersession hour job";
    job.setJobName(jobName);//from   w w w .  ja v a  2 s.c o m

    job.setJarByClass(UserSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(UserSessionHourMapper.class);
    job.setReducerClass(UserSessionHourReducer.class);

    job.setMapOutputKeyClass(UserSessionHourKey.class);
    job.setMapOutputValueClass(UserSessionHourEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(UserSessionHourPartitioner.class);
    job.setSortComparatorClass(UserSessionHourSortComparator.class);
    job.setGroupingComparatorClass(UserSessionHourGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.prerole.PreTransform.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "prerole/pretransform job";
    job.setJobName(jobName);//from w  w  w  .j  a v  a 2  s  .  c  om

    job.setJarByClass(PreTransform.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(PreTransformMapper.class);
    job.setReducerClass(PreTransformReducer.class);

    job.setMapOutputKeyClass(TransformKey.class);
    job.setMapOutputValueClass(TransformContainer.class);

    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(PreTransformPartitioner.class);
    job.setSortComparatorClass(PreTransformSortComparator.class);
    job.setGroupingComparatorClass(PreTransformGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:SecondarySort.HashToAlternateWithSS.java

protected Job jobConfig() throws IOException {
    JobConf conf = new JobConf();
    Job job = new Job(conf, "iteration");
    job.setJarByClass(HashToAlternateWithSS.class);
    job.setReducerClass(ReduceSS.class);
    job.setPartitionerClass(LongPair.HPartitioner.class);
    job.setSortComparatorClass(LongPair.Comparator.class);
    job.setGroupingComparatorClass(LongPair.GroupComparator.class);
    job.setOutputKeyClass(LongPair.class);
    job.setOutputValueClass(Text.class);
    return job;/*from w w w .java 2  s  . c o m*/
}

From source file:SecondarySort.HashToMinWithSS.java

protected Job jobConfig() throws IOException {
    JobConf conf = new JobConf();
    Job job = new Job(conf, "iteration");
    job.setJarByClass(HashToMinWithSS.class);
    job.setReducerClass(ReduceSS.class);
    job.setPartitionerClass(LongPair.HPartitioner.class);
    job.setSortComparatorClass(LongPair.Comparator.class);
    job.setGroupingComparatorClass(LongPair.GroupComparator.class);
    job.setOutputKeyClass(LongPair.class);
    job.setOutputValueClass(Text.class);
    return job;/*from   www  .  j a va  2s .c o  m*/
}

From source file:SecondarySort.inputMaker.java

License:Open Source License

protected Job jobConfig() throws IOException {
    JobConf conf = new JobConf();
    Job job = new Job(conf, "iteration");
    job.setJarByClass(inputMaker.class);
    job.setMapperClass(MapMSS.class);
    job.setReducerClass(ReduceSS.class);
    job.setPartitionerClass(LongPair.HPartitioner.class);
    job.setSortComparatorClass(LongPair.Comparator.class);
    job.setGroupingComparatorClass(LongPair.GroupComparator.class);
    job.setOutputKeyClass(LongPair.class);
    job.setOutputValueClass(Text.class);
    return job;//from w ww . j a va2s .co m
}