Example usage for org.apache.hadoop.mapreduce Job setSortComparatorClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setSortComparatorClass.

Prototype

public void setSortComparatorClass(Class<? extends RawComparator> cls) throws IllegalStateException 

Source Link

Document

Define the comparator that controls how the keys are sorted before they are passed to the Reducer .

Usage

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentDeviceStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentdevice job";
    job.setJobName(jobName);//from  w  ww.  ja  v a  2s. co m

    job.setJarByClass(ComponentDeviceStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentDeviceMapper.class);
    job.setReducerClass(ComponentDeviceReducer.class);

    job.setMapOutputKeyClass(ComponentDeviceKey.class);
    job.setMapOutputValueClass(ComponentDeviceEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentDevicePartitioner.class);
    job.setSortComparatorClass(ComponentDeviceSortComparator.class);
    job.setGroupingComparatorClass(ComponentDeviceGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentFrequencyStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componenttokenfreq job";
    job.setJobName(jobName);/*  ww  w  .ja  v  a  2  s.c om*/

    job.setJarByClass(ComponentFrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentTokenfreqMapper.class);
    job.setReducerClass(ComponentTokenfreqReducer.class);

    job.setMapOutputKeyClass(ComponentTokenfreqKey.class);
    job.setMapOutputValueClass(ComponentTokenfreqEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentTokenfreqPartitioner.class);
    job.setSortComparatorClass(ComponentTokenfreqSortComparator.class);
    job.setGroupingComparatorClass(ComponentTokenfreqGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentHourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componenthoursession job";
    job.setJobName(jobName);/*from   w  w w .j  av a2s .  co  m*/

    job.setJarByClass(ComponentHourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentHourSessionMapper.class);
    job.setReducerClass(ComponentHourSessionReducer.class);

    job.setMapOutputKeyClass(ComponentHourSessionKey.class);
    job.setMapOutputValueClass(ComponentHourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentHourSessionPartitioner.class);
    job.setSortComparatorClass(ComponentHourSessionSortComparator.class);
    job.setGroupingComparatorClass(ComponentHourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentLanguageStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentlanguage job";
    job.setJobName(jobName);//from  w  ww .j a  v  a2s  . c om

    job.setJarByClass(ComponentLanguageStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentLanguageMapper.class);
    job.setReducerClass(ComponentLanguageReducer.class);

    job.setMapOutputKeyClass(ComponentLanguageKey.class);
    job.setMapOutputValueClass(ComponentLanguageEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentLanguagePartitioner.class);
    job.setSortComparatorClass(ComponentLanguageSortComparator.class);
    job.setGroupingComparatorClass(ComponentLanguageGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentOsversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentosversion job";
    job.setJobName(jobName);//  w  w  w  .  j a  v  a 2s.co  m

    job.setJarByClass(ComponentOsversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentOsversionMapper.class);
    job.setReducerClass(ComponentOsversionReducer.class);

    job.setMapOutputKeyClass(ComponentOsversionKey.class);
    job.setMapOutputValueClass(ComponentOsversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentOsversionPartitioner.class);
    job.setSortComparatorClass(ComponentOsversionSortComparator.class);
    job.setGroupingComparatorClass(ComponentOsversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentResolutionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentresolution job";
    job.setJobName(jobName);/* ww  w  . j  av  a  2 s.  co m*/

    job.setJarByClass(ComponentResolutionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentResolutionMapper.class);
    job.setReducerClass(ComponentResolutionReducer.class);

    job.setMapOutputKeyClass(ComponentResolutionKey.class);
    job.setMapOutputValueClass(ComponentResolutionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentResolutionPartitioner.class);
    job.setSortComparatorClass(ComponentResolutionSortComparator.class);
    job.setGroupingComparatorClass(ComponentResolutionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentUserSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentusersession job";
    job.setJobName(jobName);/*from ww w  .  java 2  s .c  o m*/

    job.setJarByClass(ComponentUserSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentUserSessionMapper.class);
    job.setReducerClass(ComponentUserSessionReducer.class);

    job.setMapOutputKeyClass(ComponentUserSessionKey.class);
    job.setMapOutputValueClass(ComponentUserSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentUserSessionPartitioner.class);
    job.setSortComparatorClass(ComponentUserSessionSortComparator.class);
    job.setGroupingComparatorClass(ComponentUserSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.distribution.AppversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/appversion job";
    job.setJobName(jobName);/*from  ww  w  . j  a  va  2  s  . co m*/

    job.setJarByClass(AppversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(AppversionMapper.class);
    job.setReducerClass(AppversionReducer.class);

    job.setMapOutputKeyClass(AppversionKey.class);
    job.setMapOutputValueClass(AppversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(AppversionPartitioner.class);
    job.setSortComparatorClass(AppversionSortComparator.class);
    job.setGroupingComparatorClass(AppversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.distribution.CountryHourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/countryhoursession job";
    job.setJobName(jobName);/*from   w  ww. j  a v  a  2 s . c o  m*/

    job.setJarByClass(CountryHourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(CountryHourSessionMapper.class);
    job.setReducerClass(CountryHourSessionReducer.class);

    job.setMapOutputKeyClass(CountryHourSessionKey.class);
    job.setMapOutputValueClass(CountryHourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(CountryHourSessionPartitioner.class);
    job.setSortComparatorClass(CountryHourSessionSortComparator.class);
    job.setGroupingComparatorClass(CountryHourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

From source file:ph.fingra.hadoop.mapred.parts.distribution.CountrySessionLengthStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/countrysesstime job";
    job.setJobName(jobName);// www.  ja  v a  2  s .  com

    job.setJarByClass(CountrySessionLengthStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(CountrySesstimeMapper.class);
    job.setReducerClass(CountrySesstimeReducer.class);

    job.setMapOutputKeyClass(CountrySesstimeKey.class);
    job.setMapOutputValueClass(CountrySesstimeEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(CountrySesstimePartitioner.class);
    job.setSortComparatorClass(CountrySesstimeSortComparator.class);
    job.setGroupingComparatorClass(CountrySesstimeGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »