Example usage for org.apache.hadoop.mapred JobConf setCombinerClass

List of usage examples for org.apache.hadoop.mapred JobConf setCombinerClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setCombinerClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setCombinerClass.

Prototype

public void setCombinerClass(Class<? extends Reducer> theClass)

Source Link

Document

Set the user-defined combiner class used to combine map-outputs before being sent to the reducers.

Usage

From source file:org.sf.xrime.preprocessing.smth.Raw2OutAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2OutAdjVertexTransformer.class);
    jobConf.setJobName("Smth - OutAdjVertexesList");

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);//w ww  .j  a v a2  s  .  c o  m
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.Raw2SortedInAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2SortedInAdjVertexTransformer.class);
    jobConf.setJobName("Smth - SortedInAdjVertexesList");

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);//from  w  w  w.  j a v  a  2  s  .  c  o m
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.Raw2SortedOutAdjVertexTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Raw2SortedOutAdjVertexTransformer.class);
    jobConf.setJobName("Smth - SortedOutAdjVertexesList");

    // the keys are author names (strings)
    jobConf.setOutputKeyClass(Text.class);
    // the values are adjacent vertexes (Writable)
    jobConf.setOutputValueClass(AdjVertex.class);

    jobConf.setMapperClass(MapClass.class);
    jobConf.setCombinerClass(ReduceClass.class);
    jobConf.setReducerClass(ReduceClass.class);

    // makes the file format suitable for machine processing.
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    jobConf.setNumMapTasks(mapperNum);/*w  ww.j  ava2s  . c o  m*/
    jobConf.setNumReduceTasks(reducerNum);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        e.printStackTrace();
    }
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CheckConvergence.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*from   ww  w .j a  v a2s.  co  m*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CheckConvergenceMapper.class);
    conf.setCombinerClass(CheckConvergenceReducer.class);
    conf.setReducerClass(CheckConvergenceReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(DoubleWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(DoubleWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CountPages.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*from  w ww . j  av a  2 s  .co m*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CountPagesMapper.class);
    conf.setCombinerClass(CountPagesReducer.class);
    conf.setReducerClass(CountPagesReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(LongWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(LongWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.DanglingPages.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*from   www . j  a  v  a 2  s  .  co m*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(DanglingPagesMapper.class);
    conf.setCombinerClass(DanglingPagesReducer.class);
    conf.setReducerClass(DanglingPagesReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(DoubleWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(DoubleWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.voltdb.hadoop.mapred.VoltLoader.java

License:Open Source License

public static void main(String[] args) {

    JobClient client = new JobClient();
    JobConf conf = new JobConf(VoltLoader.class);

    VoltConfiguration.loadVoltClientJar(conf);
    new LoaderOpts(args).configure(conf);

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(VoltRecord.class);
    conf.setOutputFormat(VoltOutputFormat.class);

    conf.setMapperClass(LoadMapper.class);
    conf.setReducerClass(IdentityReducer.class);
    conf.setCombinerClass(IdentityReducer.class);

    client.setConf(conf);/*  w  w  w .j a  v  a  2  s  .c o  m*/
    try {
        JobClient.runJob(conf);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

From source file:pegasus.ResultInfo.java

License:Apache License

protected JobConf configStage2() throws Exception {
    final JobConf conf = new JobConf(getConf(), ConCmpt.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.set("cur_iter", "" + cur_iter);
    conf.set("make_symmetric", "" + make_symmetric);
    conf.setJobName("ConCmpt_Stage2");

    conf.setMapperClass(MapStage2.class);
    conf.setReducerClass(RedStage2.class);
    conf.setCombinerClass(CombinerStage2.class);

    FileInputFormat.setInputPaths(conf, tempbm_path);
    FileOutputFormat.setOutputPath(conf, nextbm_path);

    conf.setNumReduceTasks(nreducers);//w  ww.j a va  2 s  . c  o m

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(Text.class);

    return conf;
}

From source file:pegasus.ResultInfo.java

License:Apache License

protected JobConf configStage3() throws Exception {
    final JobConf conf = new JobConf(getConf(), ConCmpt.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.setJobName("ConCmpt_Stage3");

    conf.setMapperClass(MapStage3.class);
    conf.setReducerClass(RedStage3.class);
    conf.setCombinerClass(RedStage3.class);

    FileInputFormat.setInputPaths(conf, nextbm_path);
    FileOutputFormat.setOutputPath(conf, output_path);

    conf.setNumReduceTasks(1); // This is necessary.

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    return conf;/*from w ww. java2  s  . c  o m*/
}

From source file:pegasus.ResultInfo.java

License:Apache License

protected JobConf configStage4() throws Exception {
    final JobConf conf = new JobConf(getConf(), ConCmpt.class);
    conf.set("number_nodes", "" + number_nodes);
    conf.set("cur_iter", "" + cur_iter);
    conf.set("make_symmetric", "" + make_symmetric);
    conf.setJobName("ConCmpt_Stage4");

    conf.setMapperClass(MapStage4.class);
    conf.setReducerClass(RedStage4.class);
    conf.setCombinerClass(RedStage4.class);

    FileInputFormat.setInputPaths(conf, curbm_path);
    FileOutputFormat.setOutputPath(conf, summaryout_path);

    conf.setNumReduceTasks(nreducers);//  www  .j a  v a  2 s  .  co  m

    conf.setOutputKeyClass(IntWritable.class);
    conf.setOutputValueClass(IntWritable.class);

    return conf;
}