Example usage for org.apache.hadoop.mapreduce Job setPartitionerClass

List of usage examples for org.apache.hadoop.mapreduce Job setPartitionerClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setPartitionerClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setPartitionerClass.

Prototype

public void setPartitionerClass(Class<? extends Partitioner> cls) throws IllegalStateException

Source Link

Document

Set the Partitioner for the job.

Usage

From source file:ph.fingra.hadoop.mapred.parts.distribution.LanguageStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/language job";
    job.setJobName(jobName);/*from w  w  w  .ja  va 2 s. c om*/

    job.setJarByClass(LanguageStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(LanguageMapper.class);
    job.setReducerClass(LanguageReducer.class);

    job.setMapOutputKeyClass(LanguageKey.class);
    job.setMapOutputValueClass(LanguageEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(LanguagePartitioner.class);
    job.setSortComparatorClass(LanguageSortComparator.class);
    job.setGroupingComparatorClass(LanguageGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.OsversionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/osversion job";
    job.setJobName(jobName);/*from  www . jav a2s  .com*/

    job.setJarByClass(OsversionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(OsversionMapper.class);
    job.setReducerClass(OsversionReducer.class);

    job.setMapOutputKeyClass(OsversionKey.class);
    job.setMapOutputValueClass(OsversionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(OsversionPartitioner.class);
    job.setSortComparatorClass(OsversionSortComparator.class);
    job.setGroupingComparatorClass(OsversionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.distribution.ResolutionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "distribute/resolution job";
    job.setJobName(jobName);/*from w  w  w .  java 2s .c  o  m*/

    job.setJarByClass(ResolutionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(ResolutionMapper.class);
    job.setReducerClass(ResolutionReducer.class);

    job.setMapOutputKeyClass(ResolutionKey.class);
    job.setMapOutputValueClass(ResolutionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(ResolutionPartitioner.class);
    job.setSortComparatorClass(ResolutionSortComparator.class);
    job.setGroupingComparatorClass(ResolutionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.FrequencyStatistic.java

License:Apache License

public Job createJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/tokenfreq job";
    job.setJobName(jobName);/*from  w  w w.j  a  v  a2 s .c  o  m*/

    job.setJarByClass(FrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(TokenfreqMapper.class);
    job.setReducerClass(TokenfreqReducer.class);

    job.setMapOutputKeyClass(TokenfreqKey.class);
    job.setMapOutputValueClass(TokenfreqEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(TokenfreqPartitioner.class);
    job.setSortComparatorClass(TokenfreqSortComparator.class);
    job.setGroupingComparatorClass(TokenfreqGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.FrequencyStatistic.java

License:Apache License

public Job createJobFinal(Configuration conf, Path inputpath, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/frequency job";
    job.setJobName(jobName);//from   w  w w. j a v  a2s. com

    job.setJarByClass(FrequencyStatistic.class);

    FileInputFormat.addInputPath(job, inputpath);
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(FrequencyMapper.class);
    job.setCombinerClass(FrequencyReducer.class);
    job.setReducerClass(FrequencyReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(FrequencyPartitioner.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.FrequencyStatistic.java

License:Apache License

public Job createHourJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("hour", targetdate.getHour());

    Job job = new Job(conf);
    String jobName = "perform/tokenfreq hour job";
    job.setJobName(jobName);// w w  w  . j a v a 2  s.c  om

    job.setJarByClass(FrequencyStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(TokenfreqHourMapper.class);
    job.setReducerClass(TokenfreqHourReducer.class);

    job.setMapOutputKeyClass(TokenfreqHourKey.class);
    job.setMapOutputValueClass(TokenfreqHourEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(TokenfreqHourPartitioner.class);
    job.setSortComparatorClass(TokenfreqHourSortComparator.class);
    job.setGroupingComparatorClass(TokenfreqHourGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.FrequencyStatistic.java

License:Apache License

public Job createHourJobFinal(Configuration conf, Path inputpath, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/frequency hour job";
    job.setJobName(jobName);//from   w w w . j  ava 2 s .  c  om

    job.setJarByClass(FrequencyStatistic.class);

    FileInputFormat.addInputPath(job, inputpath);
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(FrequencyHourMapper.class);
    job.setCombinerClass(FrequencyHourReducer.class);
    job.setReducerClass(FrequencyHourReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(FrequencyHourPartitioner.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.HourSessionStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());

    Job job = new Job(conf);
    String jobName = "perform/hoursession job";
    job.setJobName(jobName);//from w  w w  .  jav a  2  s.co m

    job.setJarByClass(HourSessionStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(HourSessionMapper.class);
    job.setReducerClass(HourSessionReducer.class);

    job.setMapOutputKeyClass(HourSessionKey.class);
    job.setMapOutputValueClass(HourSessionEntity.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(HourSessionPartitioner.class);
    job.setSortComparatorClass(HourSessionSortComparator.class);
    job.setGroupingComparatorClass(HourSessionGroupComparator.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.NewuserStatistic.java

License:Apache License

public Job createJob(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("runmode", targetdate.getRunmode());
    conf.set("year", targetdate.getYear());
    conf.set("month", targetdate.getMonth());
    conf.set("day", targetdate.getDay());
    conf.set("week", targetdate.getWeek_str());

    Job job = new Job(conf);
    String jobName = "perform/newuser job";
    job.setJobName(jobName);//w w  w.j  av a  2 s .co m

    job.setJarByClass(NewuserStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(NewuserMapper.class);
    job.setCombinerClass(NewuserReducer.class);
    job.setReducerClass(NewuserReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);

    job.setPartitionerClass(NewuserPartitioner.class);

    job.setNumReduceTasks(numreduce);

    return job;
}

From source file:ph.fingra.hadoop.mapred.parts.performance.NewuserStatistic.java

License:Apache License

public Job createHourJobIntermediate(Configuration conf, Path[] inputpaths, Path outputpath, int numreduce,
        FingraphConfig finconfig, Path dbpath, String dbfilename, TargetDate targetdate) throws IOException {

    conf.setBoolean("verbose", finconfig.getDebug().isDebug_show_verbose());
    conf.setBoolean("counter", finconfig.getDebug().isDebug_show_counter());
    conf.set("dbfilename", dbfilename);
    conf.set("year", targetdate.getYear());
    conf.set("month", targetdate.getMonth());
    conf.set("day", targetdate.getDay());

    Job job = new Job(conf);
    String jobName = "merge/appnewuserhourlymerge hour job";
    job.setJobName(jobName);/*  w w w  . ja  va  2s .co m*/

    job.setJarByClass(NewuserStatistic.class);

    for (int i = 0; i < inputpaths.length; i++) {
        FileInputFormat.addInputPath(job, inputpaths[i]);
    }
    FileOutputFormat.setOutputPath(job, outputpath);

    job.setMapperClass(AppNewuserHourMapper.class);
    job.setCombinerClass(AppNewuserHourCombiner.class);
    job.setReducerClass(AppNewuserHourReducer.class);

    job.setMapOutputKeyClass(AppNewuserHourKey.class);
    job.setMapOutputValueClass(AppNewuserHourDb.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setPartitionerClass(AppNewuserHourPartitioner.class);

    job.setNumReduceTasks(numreduce);

    return job;
}