Example usage for org.apache.hadoop.mapreduce Job setPartitionerClass

List of usage examples for org.apache.hadoop.mapreduce Job setPartitionerClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setPartitionerClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setPartitionerClass.

Prototype

public void setPartitionerClass(Class<? extends Partitioner> cls) throws IllegalStateException

Source Link

Document

Set the Partitioner for the job.

Usage

From source file:permap.WordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: wordcount <in> <out>");
        System.exit(2);//from   ww  w . j  a  v a2  s .  c  o m
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(WordCount.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setPartitionerClass(WordPartitioner.class);
    // job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentAppversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentappversion job";
    job.setJobName(jobName);/*from   ww w .ja  v a2  s  .co  m*/

    job.setJarByClass(ComponentAppversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentAppversionMapper.class);
    job.setReducerClass(ComponentAppversionReducer.class);

    job.setMapOutputKeyClass(ComponentAppversionKey.class);
    job.setMapOutputValueClass(ComponentAppversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentAppversionPartitioner.class);
    job.setSortComparatorClass(ComponentAppversionSortComparator.class);
    job.setGroupingComparatorClass(ComponentAppversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentCountryStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentcountry job";
    job.setJobName(jobName);/*w w  w. ja  v a2  s  . c o  m*/

    job.setJarByClass(ComponentCountryStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentCountryMapper.class);
    job.setReducerClass(ComponentCountryReducer.class);

    job.setMapOutputKeyClass(ComponentCountryKey.class);
    job.setMapOutputValueClass(ComponentCountryEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentCountryPartitioner.class);
    job.setSortComparatorClass(ComponentCountrySortComparator.class);
    job.setGroupingComparatorClass(ComponentCountryGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentDeviceStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentdevice job";
    job.setJobName(jobName);//from w w w . j  av  a 2  s. co m

    job.setJarByClass(ComponentDeviceStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentDeviceMapper.class);
    job.setReducerClass(ComponentDeviceReducer.class);

    job.setMapOutputKeyClass(ComponentDeviceKey.class);
    job.setMapOutputValueClass(ComponentDeviceEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentDevicePartitioner.class);
    job.setSortComparatorClass(ComponentDeviceSortComparator.class);
    job.setGroupingComparatorClass(ComponentDeviceGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentFrequencyStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componenttokenfreq job";
    job.setJobName(jobName);//from   w  w  w. j a  va2 s . com

    job.setJarByClass(ComponentFrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentTokenfreqMapper.class);
    job.setReducerClass(ComponentTokenfreqReducer.class);

    job.setMapOutputKeyClass(ComponentTokenfreqKey.class);
    job.setMapOutputValueClass(ComponentTokenfreqEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentTokenfreqPartitioner.class);
    job.setSortComparatorClass(ComponentTokenfreqSortComparator.class);
    job.setGroupingComparatorClass(ComponentTokenfreqGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentFrequencyStatistic.java

License:Apache License

public Job createJobFinal(Configuration conf, Path inputpath, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentfrequency job";
    job.setJobName(jobName);//  w  w w  .  ja va  2s  . c o  m

    job.setJarByClass(ComponentFrequencyStatistic.class);

    FileInputFormat.addInputPath(job, inputpath);
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentFrequencyMapper.class);
    job.setCombinerClass(ComponentFrequencyReducer.class);
    job.setReducerClass(ComponentFrequencyReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentFrequencyPartitioner.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentHourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componenthoursession job";
    job.setJobName(jobName);//w w w .j a  v a2s .  com

    job.setJarByClass(ComponentHourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentHourSessionMapper.class);
    job.setReducerClass(ComponentHourSessionReducer.class);

    job.setMapOutputKeyClass(ComponentHourSessionKey.class);
    job.setMapOutputValueClass(ComponentHourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentHourSessionPartitioner.class);
    job.setSortComparatorClass(ComponentHourSessionSortComparator.class);
    job.setGroupingComparatorClass(ComponentHourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentLanguageStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentlanguage job";
    job.setJobName(jobName);/*from   w w  w  . j a v a 2  s  .  c  om*/

    job.setJarByClass(ComponentLanguageStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentLanguageMapper.class);
    job.setReducerClass(ComponentLanguageReducer.class);

    job.setMapOutputKeyClass(ComponentLanguageKey.class);
    job.setMapOutputValueClass(ComponentLanguageEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentLanguagePartitioner.class);
    job.setSortComparatorClass(ComponentLanguageSortComparator.class);
    job.setGroupingComparatorClass(ComponentLanguageGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentNewuserStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("runmode", targetdate.getRunmode());
    conf.set("year", targetdate.getYear());
    conf.set("month", targetdate.getMonth());
    conf.set("day", targetdate.getDay());
    conf.set("week", targetdate.getWeek_str());

    Job job = new Job(conf);
    String jobName = "component/componentnewuser job";
    job.setJobName(jobName);/* w w  w .j a va  2s. c  om*/

    job.setJarByClass(ComponentNewuserStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentNewuserMapper.class);
    job.setCombinerClass(ComponentNewuserReducer.class);
    job.setReducerClass(ComponentNewuserReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(ComponentNewuserPartitioner.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.component.ComponentOsversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "component/componentosversion job";
    job.setJobName(jobName);/*from ww  w  .  j a  v a  2s .c o m*/

    job.setJarByClass(ComponentOsversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ComponentOsversionMapper.class);
    job.setReducerClass(ComponentOsversionReducer.class);

    job.setMapOutputKeyClass(ComponentOsversionKey.class);
    job.setMapOutputValueClass(ComponentOsversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ComponentOsversionPartitioner.class);
    job.setSortComparatorClass(ComponentOsversionSortComparator.class);
    job.setGroupingComparatorClass(ComponentOsversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}