Example usage for org.apache.hadoop.mapreduce Job setGroupingComparatorClass

List of usage examples for org.apache.hadoop.mapreduce Job setGroupingComparatorClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setGroupingComparatorClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setGroupingComparatorClass.

Prototype

public void setGroupingComparatorClass(Class<? extends RawComparator> cls) throws IllegalStateException

Source Link

Document

Define the comparator that controls which keys are grouped together for a single call to Reducer#reduce(Object,Iterable,org.apache.hadoop.mapreduce.Reducer.Context)

Usage

From source file:pad.StarDriver.java

License:Apache License

/**
 * Execute the StarDriver Job.//  w w  w. ja v a  2  s .  c  o  m
 * @param args      array of external arguments, not used in this method
 * @return          <c>1</c> if the StarDriver Job failed its execution; <c>0</c> if everything is ok. 
 * @throws Exception 
 */
public int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    // GenericOptionsParser invocation in order to suppress the hadoop warning.
    new GenericOptionsParser(conf, args);
    conf.set("type", this.type.toString());
    Job job = new Job(conf, this.title);
    job.setJarByClass(StarDriver.class);

    job.setMapOutputKeyClass(NodesPairWritable.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(StarMapper.class);
    job.setCombinerClass(StarCombiner.class);
    job.setPartitionerClass(NodePartitioner.class);
    job.setGroupingComparatorClass(NodeGroupingComparator.class);
    job.setReducerClass(StarReducer.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    FileInputFormat.addInputPath(job, this.input);
    FileOutputFormat.setOutputPath(job, this.output);

    if (!job.waitForCompletion(verbose))
        return 1;

    // Set up the private variable looking to the counter value
    this.numChanges = job.getCounters().findCounter(UtilCounters.NUM_CHANGES).getValue();
    return 0;
}

From source file:pad.TerminationDriver.java

License:Apache License

/**
 * Execute the TerminationDriver Job./*  ww w  .  j  a  v a2s. co m*/
 * @param args      array of external arguments, not used in this method
 * @return          <c>1</c> if the TerminationDriver Job failed its execution; <c>0</c> if everything is ok. 
 * @throws Exception 
 */
public int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    // GenericOptionsParser invocation in order to suppress the hadoop warning.
    new GenericOptionsParser(conf, args);
    Job job = new Job(conf, "TerminationDriver");
    job.setJarByClass(TerminationDriver.class);

    job.setMapOutputKeyClass(NodesPairWritable.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(ClusterWritable.class);
    job.setOutputValueClass(NullWritable.class);

    job.setMapperClass(TerminationMapper.class);
    job.setPartitionerClass(NodePartitioner.class);
    job.setGroupingComparatorClass(NodeGroupingComparator.class);
    job.setReducerClass(TerminationReducer.class);

    job.setInputFormatClass(SequenceFileInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    FileInputFormat.addInputPath(job, this.input);
    FileOutputFormat.setOutputPath(job, this.output);

    if (!job.waitForCompletion(this.verbose))
        return 1;

    // Set up the private variables looking to the counters value
    this.numNodes = job.getCounters().findCounter(UtilCounters.NUM_NODES).getValue();
    this.numClusters = job.getCounters().findCounter(UtilCounters.NUM_CLUSTERS).getValue();
    return 0;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentAppversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentappversion job";
    job.setJobName(jobName);/*from  w  w  w.  j a va2  s .c o m*/

    job.setJarByClass(ComponentAppversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentAppversionMapper.class);
    job.setReducerClass(ComponentAppversionReducer.class);

    job.setMapOutputKeyClass(ComponentAppversionKey.class);
    job.setMapOutputValueClass(ComponentAppversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentAppversionPartitioner.class);
    job.setSortComparatorClass(ComponentAppversionSortComparator.class);
    job.setGroupingComparatorClass(ComponentAppversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentCountryStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentcountry job";
    job.setJobName(jobName);//  ww  w  .  jav a 2s.  c  o  m

    job.setJarByClass(ComponentCountryStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentCountryMapper.class);
    job.setReducerClass(ComponentCountryReducer.class);

    job.setMapOutputKeyClass(ComponentCountryKey.class);
    job.setMapOutputValueClass(ComponentCountryEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentCountryPartitioner.class);
    job.setSortComparatorClass(ComponentCountrySortComparator.class);
    job.setGroupingComparatorClass(ComponentCountryGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentDeviceStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentdevice job";
    job.setJobName(jobName);/*from  w  ww  .  j a va 2 s . c o  m*/

    job.setJarByClass(ComponentDeviceStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentDeviceMapper.class);
    job.setReducerClass(ComponentDeviceReducer.class);

    job.setMapOutputKeyClass(ComponentDeviceKey.class);
    job.setMapOutputValueClass(ComponentDeviceEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentDevicePartitioner.class);
    job.setSortComparatorClass(ComponentDeviceSortComparator.class);
    job.setGroupingComparatorClass(ComponentDeviceGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentFrequencyStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componenttokenfreq job";
    job.setJobName(jobName);/* w w w .j  av  a2 s.c  o  m*/

    job.setJarByClass(ComponentFrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentTokenfreqMapper.class);
    job.setReducerClass(ComponentTokenfreqReducer.class);

    job.setMapOutputKeyClass(ComponentTokenfreqKey.class);
    job.setMapOutputValueClass(ComponentTokenfreqEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentTokenfreqPartitioner.class);
    job.setSortComparatorClass(ComponentTokenfreqSortComparator.class);
    job.setGroupingComparatorClass(ComponentTokenfreqGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentHourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componenthoursession job";
    job.setJobName(jobName);/*from   w ww  .ja va  2  s .  co  m*/

    job.setJarByClass(ComponentHourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentHourSessionMapper.class);
    job.setReducerClass(ComponentHourSessionReducer.class);

    job.setMapOutputKeyClass(ComponentHourSessionKey.class);
    job.setMapOutputValueClass(ComponentHourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentHourSessionPartitioner.class);
    job.setSortComparatorClass(ComponentHourSessionSortComparator.class);
    job.setGroupingComparatorClass(ComponentHourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentLanguageStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentlanguage job";
    job.setJobName(jobName);//from w ww. j av  a  2 s. c  o  m

    job.setJarByClass(ComponentLanguageStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentLanguageMapper.class);
    job.setReducerClass(ComponentLanguageReducer.class);

    job.setMapOutputKeyClass(ComponentLanguageKey.class);
    job.setMapOutputValueClass(ComponentLanguageEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentLanguagePartitioner.class);
    job.setSortComparatorClass(ComponentLanguageSortComparator.class);
    job.setGroupingComparatorClass(ComponentLanguageGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentOsversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentosversion job";
    job.setJobName(jobName);/*from  w  ww . j a  v a  2s . c o m*/

    job.setJarByClass(ComponentOsversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentOsversionMapper.class);
    job.setReducerClass(ComponentOsversionReducer.class);

    job.setMapOutputKeyClass(ComponentOsversionKey.class);
    job.setMapOutputValueClass(ComponentOsversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentOsversionPartitioner.class);
    job.setSortComparatorClass(ComponentOsversionSortComparator.class);
    job.setGroupingComparatorClass(ComponentOsversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentResolutionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentresolution job";
    job.setJobName(jobName);//from  w  ww  . j a  v  a 2  s  .c  o  m

    job.setJarByClass(ComponentResolutionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentResolutionMapper.class);
    job.setReducerClass(ComponentResolutionReducer.class);

    job.setMapOutputKeyClass(ComponentResolutionKey.class);
    job.setMapOutputValueClass(ComponentResolutionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentResolutionPartitioner.class);
    job.setSortComparatorClass(ComponentResolutionSortComparator.class);
    job.setGroupingComparatorClass(ComponentResolutionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}