Example usage for org.apache.hadoop.mapred JobConf setMapOutputValueClass

List of usage examples for org.apache.hadoop.mapred JobConf setMapOutputValueClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobConf
  6. setMapOutputValueClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobConf setMapOutputValueClass.

Prototype

public void setMapOutputValueClass(Class<?> theClass)

Source Link

Document

Set the value class for the map output data.

Usage

From source file:org.sf.xrime.algorithms.transform.vertex.Vertex2LabeledTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf jobConf = new JobConf(conf, Vertex2LabeledTransformer.class);
    jobConf.setJobName("Vertex2Labelled");

    jobConf.setMapperClass(Vertex2LabeledMapper.class);
    jobConf.setNumReduceTasks(0);/*from w w  w  .j a  v  a2  s. co m*/
    jobConf.setOutputKeyClass(Text.class);
    if (this.theClass == null) {
        throw new ProcessorExecutionException("Need to specify the output value class.");
    }
    jobConf.setOutputValueClass(this.theClass);
    jobConf.setMapOutputKeyClass(Text.class);
    jobConf.setMapOutputValueClass(Vertex.class);
    jobConf.setInputFormat(SequenceFileInputFormat.class);
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);
    if (theLabelAdderClass != null) {
        jobConf.setClass(labelFactoryKey, theLabelAdderClass, LabelAdder.class);
    }
    FileInputFormat.setInputPaths(jobConf, srcPath);
    FileOutputFormat.setOutputPath(jobConf, destPath);

    try {
        this.runningJob = JobClient.runJob(jobConf);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.postprocessing.SequenceFileToTextFileTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf convertor = new JobConf(conf, SequenceFileToTextFileTransformer.class);
    convertor.setJobName("SequenceFileToTextFileTransformer");

    convertor.setInputFormat(SequenceFileInputFormat.class);
    convertor.setOutputFormat(TextOutputFormat.class);

    convertor.setMapperClass(SequenceFileToTextFileMapper.class);
    convertor.setMapOutputValueClass(Text.class);
    convertor.setOutputKeyClass(Text.class);
    convertor.setOutputValueClass(Text.class);

    // ONLY mapper, no combiner, no reducer.
    convertor.setNumMapTasks(getMapperNum());
    convertor.setNumReduceTasks(0);// www  .  ja  v  a2  s . c o m

    FileInputFormat.setInputPaths(convertor, srcPath);
    FileOutputFormat.setOutputPath(convertor, destPath);
    try {
        this.runningJob = JobClient.runJob(convertor);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.sf.xrime.preprocessing.smth.SmthTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf smthPreprocess = new JobConf(conf, SmthTransformer.class);
    smthPreprocess.setJobName("SmthPreprocess");

    smthPreprocess.setOutputFormat(SequenceFileOutputFormat.class);

    smthPreprocess.setMapperClass(SmthMapper.class);
    smthPreprocess.setReducerClass(SmthReducer.class);
    smthPreprocess.setNumMapTasks(mapperNum);
    smthPreprocess.setNumReduceTasks(reducerNum);

    smthPreprocess.setMapOutputValueClass(AdjVertex.class);
    smthPreprocess.setOutputKeyClass(Text.class);
    smthPreprocess.setOutputValueClass(AdjVertex.class);

    FileInputFormat.setInputPaths(smthPreprocess, srcPath);
    FileOutputFormat.setOutputPath(smthPreprocess, destPath);

    try {/*from   w  w  w .ja va2s  . c  o  m*/
        this.runningJob = JobClient.runJob(smthPreprocess);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }

}

From source file:org.sf.xrime.preprocessing.textadj.TextAdjTransformer.java

License:Apache License

@Override
public void execute() throws ProcessorExecutionException {
    JobConf textAdjPreprocess = new JobConf(conf, TextAdjTransformer.class);
    textAdjPreprocess.setJobName("SmthPreprocess");

    textAdjPreprocess.setOutputFormat(SequenceFileOutputFormat.class);

    textAdjPreprocess.setMapperClass(TextAdjMapper.class);
    textAdjPreprocess.setReducerClass(SmthReducer.class);
    textAdjPreprocess.setNumMapTasks(mapperNum);
    textAdjPreprocess.setNumReduceTasks(reducerNum);

    textAdjPreprocess.setMapOutputValueClass(AdjVertex.class);
    textAdjPreprocess.setOutputKeyClass(Text.class);
    textAdjPreprocess.setOutputValueClass(AdjVertex.class);

    FileInputFormat.setInputPaths(textAdjPreprocess, srcPath);
    FileOutputFormat.setOutputPath(textAdjPreprocess, destPath);

    try {/*  w  w w  .j av  a  2  s .  c  om*/
        this.runningJob = JobClient.runJob(textAdjPreprocess);
    } catch (IOException e) {
        throw new ProcessorExecutionException(e);
    }
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CheckConvergence.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }//from w  w  w  .  j  a v  a2 s.  c  o  m

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CheckConvergenceMapper.class);
    conf.setCombinerClass(CheckConvergenceReducer.class);
    conf.setReducerClass(CheckConvergenceReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(DoubleWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(DoubleWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CheckingData.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*from w  w  w .jav  a2 s . co m*/

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CheckingDataMapper.class);
    conf.setReducerClass(CheckingDataReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(CiteRank.NUM_REDUCE_TASKS);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.CountPages.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }//from   ww w .ja  v  a  2  s. c  o m

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(CountPagesMapper.class);
    conf.setCombinerClass(CountPagesReducer.class);
    conf.setReducerClass(CountPagesReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(LongWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(LongWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.DanglingPages.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }//from  www  . j a v  a  2 s  . c  o  m

    JobConf conf = createInputOutputConfiguration(args);

    conf.setMapperClass(DanglingPagesMapper.class);
    conf.setCombinerClass(DanglingPagesReducer.class);
    conf.setReducerClass(DanglingPagesReducer.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(DoubleWritable.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(DoubleWritable.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(1);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.InitializeRanks.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 3) {
        return usage("<input path> <output path> <number of pages>");
    }//from   w  w w.  j a v a 2s .  c o  m

    JobConf conf = createInputOutputConfiguration(args);

    conf.set(CiteRankTool.RANK_COUNT, args[2]);

    conf.setMapperClass(InitializeRanksMapper.class);

    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(Text.class);

    conf.setNumMapTasks(CiteRankTool.NUM_MAP_TASKS);
    conf.setNumReduceTasks(CiteRank.NUM_REDUCE_TASKS);

    return runJob(conf);
}

From source file:org.smartfrog.services.hadoop.benchmark.citerank.SortRanks.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 2) {
        return usage(IN_AND_OUT);
    }/*w  ww  .  j a va  2s  .  c om*/

    JobConf conf = createInputOutputConfiguration(args);
    conf.setMapperClass(SortRanksMapper.class);
    conf.setReducerClass(SortRanksReducer.class);
    conf.setOutputKeyComparatorClass(DoubleWritableDecreasingComparator.class);
    conf.setMapOutputKeyClass(DoubleWritable.class);
    conf.setMapOutputValueClass(Text.class);
    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(Text.class);
    conf.setNumReduceTasks(1); // inefficient, use InputSampler with v0.20.x
    return runJob(conf);
}