Example usage for org.apache.hadoop.mapreduce Job setGroupingComparatorClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setGroupingComparatorClass.

Prototype

public void setGroupingComparatorClass(Class<? extends RawComparator> cls) throws IllegalStateException

Source Link

Document

Define the comparator that controls which keys are grouped together for a single call to Reducer#reduce(Object,Iterable,org.apache.hadoop.mapreduce.Reducer.Context)

Usage

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentUserSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentusersession job";
    job.setJobName(jobName);/*w ww. ja  va2s.  c o m*/

    job.setJarByClass(ComponentUserSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentUserSessionMapper.class);
    job.setReducerClass(ComponentUserSessionReducer.class);

    job.setMapOutputKeyClass(ComponentUserSessionKey.class);
    job.setMapOutputValueClass(ComponentUserSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentUserSessionPartitioner.class);
    job.setSortComparatorClass(ComponentUserSessionSortComparator.class);
    job.setGroupingComparatorClass(ComponentUserSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.AppversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/appversion job";
    job.setJobName(jobName);// ww  w  .j a  v  a  2s . c  o m

    job.setJarByClass(AppversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(AppversionMapper.class);
    job.setReducerClass(AppversionReducer.class);

    job.setMapOutputKeyClass(AppversionKey.class);
    job.setMapOutputValueClass(AppversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(AppversionPartitioner.class);
    job.setSortComparatorClass(AppversionSortComparator.class);
    job.setGroupingComparatorClass(AppversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.CountryHourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/countryhoursession job";
    job.setJobName(jobName);//from  ww w  . j a  v a 2 s . c  om

    job.setJarByClass(CountryHourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(CountryHourSessionMapper.class);
    job.setReducerClass(CountryHourSessionReducer.class);

    job.setMapOutputKeyClass(CountryHourSessionKey.class);
    job.setMapOutputValueClass(CountryHourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(CountryHourSessionPartitioner.class);
    job.setSortComparatorClass(CountryHourSessionSortComparator.class);
    job.setGroupingComparatorClass(CountryHourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.CountrySessionLengthStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/countrysesstime job";
    job.setJobName(jobName);/* w  ww.j  av a 2s.com*/

    job.setJarByClass(CountrySessionLengthStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(CountrySesstimeMapper.class);
    job.setReducerClass(CountrySesstimeReducer.class);

    job.setMapOutputKeyClass(CountrySesstimeKey.class);
    job.setMapOutputValueClass(CountrySesstimeEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(CountrySesstimePartitioner.class);
    job.setSortComparatorClass(CountrySesstimeSortComparator.class);
    job.setGroupingComparatorClass(CountrySesstimeGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.CountryStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/country job";
    job.setJobName(jobName);/*from ww w .  j  a va  2  s . c om*/

    job.setJarByClass(CountryStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(CountryMapper.class);
    job.setReducerClass(CountryReducer.class);

    job.setMapOutputKeyClass(CountryKey.class);
    job.setMapOutputValueClass(CountryEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(CountryPartitioner.class);
    job.setSortComparatorClass(CountrySortComparator.class);
    job.setGroupingComparatorClass(CountryGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.DeviceStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/device job";
    job.setJobName(jobName);//  w  w  w .j av  a  2  s . c o  m

    job.setJarByClass(DeviceStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(DeviceMapper.class);
    job.setReducerClass(DeviceReducer.class);

    job.setMapOutputKeyClass(DeviceKey.class);
    job.setMapOutputValueClass(DeviceEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(DevicePartitioner.class);
    job.setSortComparatorClass(DeviceSortComparator.class);
    job.setGroupingComparatorClass(DeviceGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.LanguageStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/language job";
    job.setJobName(jobName);/*from   w  w  w  . j av  a  2 s.c o  m*/

    job.setJarByClass(LanguageStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);

    job.setMapOutputKeyClass(LanguageKey.class);
    job.setMapOutputValueClass(LanguageEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(LanguagePartitioner.class);
    job.setSortComparatorClass(LanguageSortComparator.class);
    job.setGroupingComparatorClass(LanguageGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.OsversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/osversion job";
    job.setJobName(jobName);//from ww w. jav a2 s.co  m

    job.setJarByClass(OsversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(OsversionMapper.class);
    job.setReducerClass(OsversionReducer.class);

    job.setMapOutputKeyClass(OsversionKey.class);
    job.setMapOutputValueClass(OsversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(OsversionPartitioner.class);
    job.setSortComparatorClass(OsversionSortComparator.class);
    job.setGroupingComparatorClass(OsversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.ResolutionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/resolution job";
    job.setJobName(jobName);//w  ww.j  a v a 2  s.c o m

    job.setJarByClass(ResolutionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ResolutionMapper.class);
    job.setReducerClass(ResolutionReducer.class);

    job.setMapOutputKeyClass(ResolutionKey.class);
    job.setMapOutputValueClass(ResolutionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ResolutionPartitioner.class);
    job.setSortComparatorClass(ResolutionSortComparator.class);
    job.setGroupingComparatorClass(ResolutionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.FrequencyStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/tokenfreq job";
    job.setJobName(jobName);/*from  w ww.  ja  va2 s .  co  m*/

    job.setJarByClass(FrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(TokenfreqMapper.class);
    job.setReducerClass(TokenfreqReducer.class);

    job.setMapOutputKeyClass(TokenfreqKey.class);
    job.setMapOutputValueClass(TokenfreqEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(TokenfreqPartitioner.class);
    job.setSortComparatorClass(TokenfreqSortComparator.class);
    job.setGroupingComparatorClass(TokenfreqGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}