List of usage examples for org.apache.hadoop.mapreduce Job setMapperClass
public void setMapperClass(Class<? extends Mapper> cls) throws IllegalStateException
From source file:com.javiertordable.mrif.MapReduceQuadraticSieve.java
License:Apache License
/** * Setup the MapReduce parameters and run it. * * Tool parses the command line arguments for us. *///from w w w . ja va2 s . c o m public int run(String[] args) throws Exception { Configuration conf = getConf(); // Check the arguments. we need the integer to attempt to factor. if (args.length < 1) { System.out.println("Please indicate the integer to factor"); LOGGER.severe("No integer to factor. Exit."); System.exit(1); } // Parse N and add it to the job configuration, so that the workers can // access it as well. BigInteger N = new BigInteger(args[0]); LOGGER.info("Attempting factorization of: " + N.toString()); conf.set(INTEGER_TO_FACTOR_NAME, N.toString()); // Obtain the factor base for the integer N. FactorBaseArray factorBase = SieveInput.factorBase(N); LOGGER.info("Factor base of size: " + factorBase.size()); conf.set(FACTOR_BASE_NAME, factorBase.toString()); // Prepare the input of the mapreduce. LOGGER.info("Sieve of size: " + SieveInput.fullSieveIntervalSize(N)); try { // Write the full sieve interval to disk. SieveInput.writeFullSieveInterval(N, "input/" + INPUT_FILE_NAME); } catch (FileNotFoundException e) { System.out.println("Unable to open the file for writing."); } catch (IOException e) { System.out.println("Unable to write to the output file."); } // Configure the classes of the mapreducer Job job = new Job(conf, "QuadraticSieve"); job.setJarByClass(MapReduceQuadraticSieve.class); job.setMapperClass(SieveMapper.class); job.setReducerClass(FindSquaresReducer.class); // Output will be two pairs of strings: // <"Factor1", "59"> // <"Factor2", "101"> job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path("input/")); FileOutputFormat.setOutputPath(job, new Path("output/")); // Submit the job. job.waitForCompletion(true); return 0; }
From source file:com.jbw.mutioutputformat.PatitionByStation.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path input = new Path(conf.get("input")); Path output = new Path(conf.get("output")); Job job = Job.getInstance(); job.setJarByClass(PatitionByStation.class); job.setJobName("papapa"); job.setMapperClass(StationMapper.class); job.setMapOutputKeyClass(Text.class); job.setReducerClass(StationReducer.class); job.setOutputKeyClass(NullWritable.class); FileInputFormat.addInputPath(job, input); FileOutputFormat.setOutputPath(job, output); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.add.AddMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job job = Job.getInstance(conf); job.setJobName("5"); job.setJarByClass(AddMRD.class); job.setMapperClass(AddMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setReducerClass(AddReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(job, in); TextOutputFormat.setOutputPath(job, out); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.cooc.CoocMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job surJob = Job.getInstance(conf); surJob.setJarByClass(CoocMRD.class); surJob.setJobName("Coor"); surJob.setMapperClass(CoocMapper.class); surJob.setReducerClass(IntSumReducer.class); surJob.setMapOutputKeyClass(Text.class); surJob.setMapOutputValueClass(IntWritable.class); surJob.setOutputKeyClass(Text.class); surJob.setOutputValueClass(IntWritable.class); surJob.setInputFormatClass(TextInputFormat.class); surJob.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(surJob, in); TextOutputFormat.setOutputPath(surJob, out); return surJob.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.iteamlist.ItemMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job surJob = Job.getInstance(conf); surJob.setJarByClass(ItemMRD.class); surJob.setJobName("item"); surJob.setMapperClass(ItemMapper.class); surJob.setReducerClass(ItemReducer.class); surJob.setMapOutputKeyClass(Text.class); surJob.setMapOutputValueClass(Text.class); surJob.setOutputKeyClass(Text.class); surJob.setOutputValueClass(Text.class); surJob.setInputFormatClass(TextInputFormat.class); surJob.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(surJob, in); TextOutputFormat.setOutputPath(surJob, out); return surJob.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.martrixlist.MartrixListMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job surJob = Job.getInstance(conf); surJob.setJarByClass(MartrixListMRD.class); surJob.setJobName("user"); surJob.setMapperClass(MListMapper.class); surJob.setReducerClass(MListReducer.class); surJob.setMapOutputKeyClass(Text.class); surJob.setMapOutputValueClass(Text.class); surJob.setOutputKeyClass(Text.class); surJob.setOutputValueClass(Text.class); surJob.setInputFormatClass(TextInputFormat.class); surJob.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(surJob, in); TextOutputFormat.setOutputPath(surJob, out); return surJob.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.tar.sf.TarDriver.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job job = Job.getInstance(conf); job.setJarByClass(TarDriver.class); job.setJobName("test"); job.setMapperClass(SmallFileMapper.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(TextInputFormat.class); TextInputFormat.addInputPath(job, in); job.setOutputFormatClass(TarOutputFormat.class); TarOutputFormat.setOutputPath(job, out); // job.setNumReduceTasks(0); /*//ww w . j a va 2s . c om job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TarOutputFormat.class); job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(Text.class); */ return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jeffy.mr.WordCount.java
License:Apache License
/** * @param args// ww w .j a va 2 s . co m */ public static void main(String[] args) { String input = "hdfs://master:8020/tmp/jeffy/input/wordcount.txt"; String output = "hdfs://master:8020/tmp/jeffy/output"; Configuration config = new Configuration(); /** * Windows???no jobCtrol * http://stackoverflow.com/questions/24075669/mapreduce-job-fail-when-submitted-from-windows-machine */ config.set("mapreduce.app-submission.cross-platform", "true"); config.set("mapred.remote.os", "Linux"); try { Job job = Job.getInstance(config); //Windows??? job.setJarByClass(WordCount.class); //????? job.setJar("D:\\bigdata\\mapreduce-demo\\src\\main\\java\\WordCount.jar"); job.setJobName("Wordcount job"); job.setMapperClass(WordCountMapper.class); job.setReducerClass(WordCountReducer.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); TextInputFormat.setInputPaths(job, new Path(input)); TextOutputFormat.setOutputPath(job, new Path(output)); // Submit the job, then poll for progress until the job is complete try { job.waitForCompletion(true); } catch (ClassNotFoundException | InterruptedException e) { e.printStackTrace(); } } catch (IOException e) { e.printStackTrace(); } }
From source file:com.jet.hadoop.wordcount.WordCount.java
License:Apache License
public static void main(String[] args) throws Exception { long timeBegin = System.currentTimeMillis(); System.out.println("hadoop wordcount begins at" + timeBegin); if (args == null || args.length == 0) { args = new String[2]; args[0] = "E:\\Work\\input\\hello.txt"; args[1] = "E:\\Work\\output"; }//from w w w. ja v a2 s. c om Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { System.err.println("Usage: wordcount <in> [<in>...] <out>"); System.exit(2); } Job job = Job.getInstance(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); // job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); // job.setNumReduceTasks(2); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); for (int i = 0; i < otherArgs.length - 1; ++i) { FileInputFormat.addInputPath(job, new Path(otherArgs[i])); } FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1])); boolean result = job.waitForCompletion(true); long timeEnd = System.currentTimeMillis(); System.out.println("hadoop wordcount ended at" + timeEnd); System.out.println("hadoop wordcount cost time" + (timeEnd - timeBegin) / 1000 + " seconds."); System.exit(result ? 0 : 1); }
From source file:com.jhkt.playgroundArena.hadoop.tasks.jobs.AverageJob.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); Job job = new Job(conf, AverageJob.class.getSimpleName()); job.setJarByClass(AverageJob.class); Path in = new Path(args[0]); Path out = new Path(args[1]); FileInputFormat.setInputPaths(job, in); FileOutputFormat.setOutputPath(job, out); job.setJobName("Sample Average Job"); job.setMapperClass(AverageMapper.class); job.setCombinerClass(AverageCombiner.class); job.setReducerClass(AverageReducer.class); job.setInputFormatClass(TextInputFormat.class); //job.setOutputFormatClass(TextOutputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); FileOutputFormat.setCompressOutput(job, true); FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(IntWritable.class); System.exit(job.waitForCompletion(true) ? 0 : 1); return 0;/*w ww .jav a2s . c o m*/ }