Example usage for org.apache.hadoop.mapreduce Job setOutputKeyClass

List of usage examples for org.apache.hadoop.mapreduce Job setOutputKeyClass

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapreduce.*
  5. Job
  6. setOutputKeyClass

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce Job setOutputKeyClass.

Prototype

public void setOutputKeyClass(Class<?> theClass) throws IllegalStateException

Source Link

Document

Set the key class for the job output data.

Usage

From source file:com.intel.hadoop.hbase.dot.KEY.java

License:Apache License

private void doMapReduce(Class<? extends InputFormat> inputFormatClass, Class<? extends Mapper> mapperClass,
        String mrTableName) throws IOException, ClassNotFoundException, InterruptedException {

    this.conf.set(KEY.INPUT_TABLE, mrTableName);
    Job job = new Job(this.conf);
    job.setJobName("Generate Data for [" + mrTableName + "]");
    job.setJarByClass(GenerateTestTable.class);

    job.setInputFormatClass(inputFormatClass);

    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(LongWritable.class);

    FileSystem fs = FileSystem.get(conf);
    Path path = new Path("/tmp", "tempout");
    fs.delete(path, true);//from  w  ww  .j  a  v  a 2s  . c  om

    FileOutputFormat.setOutputPath(job, path);

    job.setMapperClass(mapperClass);
    job.setNumReduceTasks(0);

    TableMapReduceUtil.addDependencyJars(job);
    // Add a Class from the hbase.jar so it gets registered too.
    TableMapReduceUtil.addDependencyJars(job.getConfiguration(), org.apache.hadoop.hbase.util.Bytes.class);

    TableMapReduceUtil.initCredentials(job);

    job.waitForCompletion(true);

}

From source file:com.j.distributed.counter.CounterJob.java

@Override
public int run(String... options) throws Exception {

    Job job = Job.getInstance(getConf(), getClass().toString());
    job.setJarByClass(getClass());//from  w w w .  j a va 2s.co  m

    job.setMapperClass(CounterMapper.class);
    job.setCombinerClass(CounterReducer.class);
    job.setReducerClass(CounterReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    FileInputFormat.addInputPath(job, new Path(options[0]));
    FileOutputFormat.setOutputPath(job, new Path(options[1]));
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.j.distributed.sorter.SorterJob.java

@Override
public int run(String... options) throws Exception {

    Job job = Job.getInstance(getConf(), getClass().toString());
    job.setJarByClass(getClass());//w  ww  . ja v  a 2s.c  om

    job.setMapperClass(SorterMapper.class);
    job.setCombinerClass(SorterReducer.class);
    job.setReducerClass(SorterReducer.class);

    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);
    job.setSortComparatorClass(LongWritable.DecreasingComparator.class);

    FileInputFormat.addInputPath(job, new Path(options[1]));
    FileOutputFormat.setOutputPath(job, new Path(options[2]));
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.javiertordable.mrif.MapReduceQuadraticSieve.java

License:Apache License

/**
 * Setup the MapReduce parameters and run it.
 *
 * Tool parses the command line arguments for us.
 *///from  ww w .  ja v  a2  s .c  om
public int run(String[] args) throws Exception {
    Configuration conf = getConf();

    // Check the arguments. we need the integer to attempt to factor.
    if (args.length < 1) {
        System.out.println("Please indicate the integer to factor");
        LOGGER.severe("No integer to factor. Exit.");
        System.exit(1);
    }

    // Parse N and add it to the job configuration, so that the workers can
    // access it as well.
    BigInteger N = new BigInteger(args[0]);
    LOGGER.info("Attempting factorization of: " + N.toString());
    conf.set(INTEGER_TO_FACTOR_NAME, N.toString());

    // Obtain the factor base for the integer N.
    FactorBaseArray factorBase = SieveInput.factorBase(N);
    LOGGER.info("Factor base of size: " + factorBase.size());
    conf.set(FACTOR_BASE_NAME, factorBase.toString());

    // Prepare the input of the mapreduce.
    LOGGER.info("Sieve of size: " + SieveInput.fullSieveIntervalSize(N));
    try {
        // Write the full sieve interval to disk.
        SieveInput.writeFullSieveInterval(N, "input/" + INPUT_FILE_NAME);
    } catch (FileNotFoundException e) {
        System.out.println("Unable to open the file for writing.");
    } catch (IOException e) {
        System.out.println("Unable to write to the output file.");
    }

    // Configure the classes of the mapreducer
    Job job = new Job(conf, "QuadraticSieve");
    job.setJarByClass(MapReduceQuadraticSieve.class);
    job.setMapperClass(SieveMapper.class);
    job.setReducerClass(FindSquaresReducer.class);

    // Output will be two pairs of strings:
    // <"Factor1", "59">
    // <"Factor2", "101">
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    FileInputFormat.addInputPath(job, new Path("input/"));
    FileOutputFormat.setOutputPath(job, new Path("output/"));

    // Submit the job.
    job.waitForCompletion(true);

    return 0;
}

From source file:com.jbw.mutioutputformat.PatitionByStation.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path input = new Path(conf.get("input"));
    Path output = new Path(conf.get("output"));
    Job job = Job.getInstance();
    job.setJarByClass(PatitionByStation.class);
    job.setJobName("papapa");
    job.setMapperClass(StationMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setReducerClass(StationReducer.class);
    job.setOutputKeyClass(NullWritable.class);
    FileInputFormat.addInputPath(job, input);
    FileOutputFormat.setOutputPath(job, output);
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.add.AddMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job job = Job.getInstance(conf);
    job.setJobName("5");
    job.setJarByClass(AddMRD.class);

    job.setMapperClass(AddMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(AddReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(job, in);
    TextOutputFormat.setOutputPath(job, out);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.cooc.CoocMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(CoocMRD.class);
    surJob.setJobName("Coor");

    surJob.setMapperClass(CoocMapper.class);
    surJob.setReducerClass(IntSumReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(IntWritable.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(IntWritable.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.filter.FilterMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path rPath = new Path(conf.get("rin"));
    Path aPath = new Path(conf.get("ain"));
    Path out = new Path(conf.get("out"));

    Job job = Job.getInstance(conf);
    job.setJobName("666");
    job.setJarByClass(FilterMRD.class);

    MultipleInputs.addInputPath(job, rPath, TextInputFormat.class, RelationMapper.class);
    MultipleInputs.addInputPath(job, aPath, TextInputFormat.class, AddMapper.class);
    job.setMapOutputKeyClass(Text.class);

    job.setReducerClass(FilterReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, out);

    job.setOutputKeyClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.guiyihua.GYHMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("in"));
    Path out = new Path(conf.get("out"));

    Job job = Job.getInstance(conf);
    job.setJarByClass(GYHMRD.class);
    job.setJobName("fdsjh");
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setInputFormatClass(TextInputFormat.class);

    job.setPartitionerClass(XXPartition.class);

    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    job.setNumReduceTasks(2);//w ww .j ava 2  s.  co m

    TextInputFormat.addInputPath(job, in);
    TextOutputFormat.setOutputPath(job, out);
    return job.waitForCompletion(true) ? 0 : 1;
}

From source file:com.jbw.recommendsystem.iteamlist.ItemMRD.java

@Override
public int run(String[] strings) throws Exception {
    Configuration conf = getConf();
    Path in = new Path(conf.get("input"));
    Path out = new Path(conf.get("output"));

    Job surJob = Job.getInstance(conf);
    surJob.setJarByClass(ItemMRD.class);
    surJob.setJobName("item");

    surJob.setMapperClass(ItemMapper.class);
    surJob.setReducerClass(ItemReducer.class);

    surJob.setMapOutputKeyClass(Text.class);
    surJob.setMapOutputValueClass(Text.class);

    surJob.setOutputKeyClass(Text.class);
    surJob.setOutputValueClass(Text.class);

    surJob.setInputFormatClass(TextInputFormat.class);
    surJob.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(surJob, in);
    TextOutputFormat.setOutputPath(surJob, out);

    return surJob.waitForCompletion(true) ? 0 : 1;
}