Example usage for org.apache.hadoop.mapred JobConf setMapperClass

List of usage examples for org.apache.hadoop.mapred JobConf setMapperClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setMapperClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setMapperClass.

Prototype

public void setMapperClass(Class<? extends Mapper> theClass)

Source Link

Document

Set the Mapper class for the job.

Usage

From source file:SBP.java

License:Apache License

protected JobConf configInitMessage() throws Exception {
    final JobConf conf = new JobConf(getConf(), SBP.class);
    conf.set("nstate", "" + nstate);
    conf.set("compat_matrix_str", "" + edge_potential_str);
    conf.setJobName("BP_Init_Belief");

    conf.setMapperClass(MapInitializeBelief.class);

    fs.delete(message_cur_path, true);/*from w ww  .ja v  a  2  s.  c  o m*/

    FileInputFormat.setInputPaths(conf, edge_path);
    FileOutputFormat.setOutputPath(conf, message_cur_path);

    conf.setNumReduceTasks(0);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);
    return conf;
}

From source file:SBP.java

License:Apache License

protected JobConf configUpdateMessage() throws Exception {
    final JobConf conf = new JobConf(getConf(), SBP.class);
    conf.set("nstate", "" + nstate);
    conf.set("compat_matrix_str", "" + edge_potential_str);
    conf.setJobName("BP_Update_message");

    conf.setMapperClass(MapUpdateMessage.class);
    conf.setReducerClass(RedUpdateMessage.class);

    fs.delete(message_next_path, true);/*from  w w w.j a va2 s .  co m*/

    FileInputFormat.setInputPaths(conf, message_cur_path, prior_path);
    FileOutputFormat.setOutputPath(conf, message_next_path);

    conf.setNumReduceTasks(nreducer);

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:SBP.java

License:Apache License

protected JobConf configSmoothMessage() throws Exception {
    final JobConf conf = new JobConf(getConf(), SBP.class);
    conf.set("smooth_lambda", "" + lambda);
    conf.set("nstate", "" + nstate);
    conf.set("compat_matrix_str", "" + edge_potential_str);
    conf.setJobName("BP_Smooth_message");

    fs.delete(message_smooth_path, true);

    conf.setMapperClass(MapSmoothMessage.class);
    conf.setReducerClass(RedSmoothMessage.class);

    FileInputFormat.setInputPaths(conf, message_cur_path, message_next_path);
    FileOutputFormat.setOutputPath(conf, message_smooth_path);

    conf.setNumReduceTasks(nreducer);//  w w w  .  j ava 2s. c o m

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:SBP.java

License:Apache License

protected JobConf configCheckErr() throws Exception {
    final JobConf conf = new JobConf(getConf(), SBP.class);
    conf.set("nstate", "" + nstate);
    conf.setJobName("BP_Check Err");

    fs.delete(check_error_path, true);//from  w  ww .jav a2  s  .  c o  m

    conf.setMapperClass(MapCheckErr.class);
    conf.setReducerClass(RedCheckErr.class);

    FileInputFormat.setInputPaths(conf, message_cur_path, message_smooth_path);
    FileOutputFormat.setOutputPath(conf, check_error_path);

    conf.setNumReduceTasks(nreducer);

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:SBP.java

License:Apache License

protected JobConf configSumErr() throws Exception {
    final JobConf conf = new JobConf(getConf(), SBP.class);
    conf.set("nstate", "" + nstate);
    conf.setJobName("BP_Sum Err");

    fs.delete(sum_error_path, true);/*  w  w  w . ja v a 2 s  .  c  o m*/

    conf.setMapperClass(MapSumErr.class);
    conf.setReducerClass(RedSumErr.class);

    FileInputFormat.setInputPaths(conf, check_error_path);
    FileOutputFormat.setOutputPath(conf, sum_error_path);

    conf.setNumReduceTasks(1);

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:SBP.java

License:Apache License

protected JobConf configComputeBelief() throws Exception {
    final JobConf conf = new JobConf(getConf(), SBP.class);
    conf.set("nstate", "" + nstate);
    conf.set("compat_matrix_str", "" + edge_potential_str);
    conf.setJobName("BP_Compute_Belief");

    conf.setMapperClass(MapComputeBelief.class);
    conf.setReducerClass(RedComputeBelief.class);

    fs.delete(output_path, true);//from  w w  w  .  j a va  2  s  . c  om

    FileInputFormat.setInputPaths(conf, message_cur_path, prior_path);
    FileOutputFormat.setOutputPath(conf, output_path);

    conf.setNumReduceTasks(nreducer);

    conf.setOutputKeyClass(LongWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:gen_rank.java

License:LGPL

public static void runjob(String input, String output) throws Exception {
    JobConf conf = new JobConf(gen_rank.class);
    conf.setJobName("Preparing_data");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapperClass(Map.class);
    conf.setCombinerClass(Reduce.class);
    conf.setReducerClass(Reduce.class);

    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(input));
    FileOutputFormat.setOutputPath(conf, new Path(output));

    JobClient.runJob(conf);// w w w . j  a v  a  2s .  c  o m
}

From source file:GapDeduceRunner.java

License:Apache License

public static void main(String[] args) throws IOException {
    JobConf conf = new JobConf(GapDeduceRunner.class);
    conf.setJobName("gapdeduce");

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setMapperClass(Gapper.class);
    conf.setReducerClass(Deducer.class);

    // KeyValueTextInputFormat treats each line as an input record, 
    // and splits the line by the tab character to separate it into key and value 
    conf.setInputFormat(KeyValueTextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    JobClient.runJob(conf);//w  w  w.j a v  a  2 s.c om
}

From source file:LicenseStatewiseCount.java

public static void main(String[] args) throws Exception {

    JobConf conf = new JobConf(ParkingTicketStatewiseCount.class);
    conf.setJobName("Statecounts");

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(IntWritable.class);

    conf.setMapperClass(Map.class);
    conf.setReducerClass(Reduce.class);

    conf.setInputFormat(TextInputFormat.class);
    conf.setOutputFormat(TextOutputFormat.class);

    FileInputFormat.setInputPaths(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    JobClient.runJob(conf);/* w w  w  .j a  va2 s  .  c  o  m*/

}

From source file:DijikstraAlgo.java

License:GNU General Public License

public static void run(String[] args) throws Exception {
    IN = "hdfs://10.8.3.161:9000/user/sagar/input/";
    OUT = "hdfs://10.8.3.161:9000/user/sagar/output/";
    String input = IN;/* w  w w. j av a 2  s  .  com*/
    String output = OUT + System.nanoTime();
    String MAX_SPLIT_SIZE = args[0];
    boolean isdone = false;

    // Reiteration again and again till the convergence
    while (isdone == false) {
        JobConf conf = new JobConf(DijikstraAlgo.class);
        conf.setJobName("Dijikstra");
        // conf.set("mapred.max.split.size", MAX_SPLIT_SIZE);
        conf.setOutputKeyClass(LongWritable.class);
        conf.setOutputValueClass(Text.class);
        conf.setMapperClass(Map.class);
        conf.setReducerClass(Reduce.class);
        conf.setInputFormat(TextInputFormat.class);
        conf.setOutputFormat(TextOutputFormat.class);

        FileInputFormat.setInputPaths(conf, new Path(input));
        FileOutputFormat.setOutputPath(conf, new Path(output));

        JobClient.runJob(conf);

        input = output + "/part-00000";
        isdone = true;// set the job to NOT run again!
        Path ofile = new Path(input);
        FileSystem fs = FileSystem.get(new URI("hdfs://10.8.3.165:9000"), conf);
        //FileSystem fs = FileSystem.get(new Configuration());
        BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(ofile)));
        HashMap<Integer, Integer> imap = new HashMap<Integer, Integer>();
        String line = br.readLine();
        // Read the current output file and put it into HashMap
        while (line != null) {
            String[] sp = line.split("\t| ");
            int node = Integer.parseInt(sp[0]);
            int distance = Integer.parseInt(sp[1]);
            imap.put(node, distance);
            line = br.readLine();
        }
        br.close();

        // Check for convergence condition if any node is still left then
        // continue else stop
        Iterator<Integer> itr = imap.keySet().iterator();
        while (itr.hasNext()) {
            int key = itr.next();
            int value = imap.get(key);
            if (value >= 125) {
                isdone = false;
            }
        }
        input = output;
        output = OUT + System.nanoTime();
    }
}