List of usage examples for org.apache.hadoop.mapreduce Job setOutputFormatClass
public void setOutputFormatClass(Class<? extends OutputFormat> cls) throws IllegalStateException
From source file:com.inmobi.databus.local.LocalStreamService.java
License:Apache License
private Job createJob(Path inputPath) throws IOException { String jobName = "localstream"; Configuration conf = cluster.getHadoopConf(); Job job = new Job(conf); job.setJobName(jobName);/*from w w w .ja va2s.c o m*/ KeyValueTextInputFormat.setInputPaths(job, inputPath); job.setInputFormatClass(KeyValueTextInputFormat.class); job.setJarByClass(CopyMapper.class); job.setMapperClass(CopyMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(NullOutputFormat.class); job.getConfiguration().set("mapred.map.tasks.speculative.execution", "false"); job.getConfiguration().set("localstream.tmp.path", tmpPath.toString()); return job; }
From source file:com.jbw.recommendsystem.add.AddMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job job = Job.getInstance(conf); job.setJobName("5"); job.setJarByClass(AddMRD.class); job.setMapperClass(AddMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setReducerClass(AddReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(job, in); TextOutputFormat.setOutputPath(job, out); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.cooc.CoocMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job surJob = Job.getInstance(conf); surJob.setJarByClass(CoocMRD.class); surJob.setJobName("Coor"); surJob.setMapperClass(CoocMapper.class); surJob.setReducerClass(IntSumReducer.class); surJob.setMapOutputKeyClass(Text.class); surJob.setMapOutputValueClass(IntWritable.class); surJob.setOutputKeyClass(Text.class); surJob.setOutputValueClass(IntWritable.class); surJob.setInputFormatClass(TextInputFormat.class); surJob.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(surJob, in); TextOutputFormat.setOutputPath(surJob, out); return surJob.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.filter.FilterMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path rPath = new Path(conf.get("rin")); Path aPath = new Path(conf.get("ain")); Path out = new Path(conf.get("out")); Job job = Job.getInstance(conf); job.setJobName("666"); job.setJarByClass(FilterMRD.class); MultipleInputs.addInputPath(job, rPath, TextInputFormat.class, RelationMapper.class); MultipleInputs.addInputPath(job, aPath, TextInputFormat.class, AddMapper.class); job.setMapOutputKeyClass(Text.class); job.setReducerClass(FilterReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, out); job.setOutputKeyClass(Text.class); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.guiyihua.GYHMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("in")); Path out = new Path(conf.get("out")); Job job = Job.getInstance(conf); job.setJarByClass(GYHMRD.class); job.setJobName("fdsjh"); job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(Text.class); job.setInputFormatClass(TextInputFormat.class); job.setPartitionerClass(XXPartition.class); job.setOutputFormatClass(TextOutputFormat.class); job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(2);/*from w ww.j av a 2 s . c o m*/ TextInputFormat.addInputPath(job, in); TextOutputFormat.setOutputPath(job, out); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.iteamlist.ItemMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job surJob = Job.getInstance(conf); surJob.setJarByClass(ItemMRD.class); surJob.setJobName("item"); surJob.setMapperClass(ItemMapper.class); surJob.setReducerClass(ItemReducer.class); surJob.setMapOutputKeyClass(Text.class); surJob.setMapOutputValueClass(Text.class); surJob.setOutputKeyClass(Text.class); surJob.setOutputValueClass(Text.class); surJob.setInputFormatClass(TextInputFormat.class); surJob.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(surJob, in); TextOutputFormat.setOutputPath(surJob, out); return surJob.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.joinim.JoinMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path itemPath = new Path(conf.get("iteminput")); Path matrixPath = new Path(conf.get("matrixinput")); Path out = new Path(conf.get("output")); Job job = Job.getInstance(conf); job.setJobName("jjj"); job.setJarByClass(JoinMRD.class); MultipleInputs.addInputPath(job, itemPath, TextInputFormat.class, IteamMapper.class); MultipleInputs.addInputPath(job, matrixPath, TextInputFormat.class, MatrixMapper.class); job.setMapOutputKeyClass(Text.class); job.setReducerClass(JoinReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, out); job.setOutputKeyClass(Text.class); return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.recommendsystem.martrixlist.MartrixListMRD.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job surJob = Job.getInstance(conf); surJob.setJarByClass(MartrixListMRD.class); surJob.setJobName("user"); surJob.setMapperClass(MListMapper.class); surJob.setReducerClass(MListReducer.class); surJob.setMapOutputKeyClass(Text.class); surJob.setMapOutputValueClass(Text.class); surJob.setOutputKeyClass(Text.class); surJob.setOutputValueClass(Text.class); surJob.setInputFormatClass(TextInputFormat.class); surJob.setOutputFormatClass(TextOutputFormat.class); TextInputFormat.addInputPath(surJob, in); TextOutputFormat.setOutputPath(surJob, out); return surJob.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.tar.sf.TarDriver.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job job = Job.getInstance(conf); job.setJarByClass(TarDriver.class); job.setJobName("test"); job.setMapperClass(SmallFileMapper.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(TextInputFormat.class); TextInputFormat.addInputPath(job, in); job.setOutputFormatClass(TarOutputFormat.class); TarOutputFormat.setOutputPath(job, out); // job.setNumReduceTasks(0); /*//from w w w .j a v a 2 s .c o m job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TarOutputFormat.class); job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(Text.class); */ return job.waitForCompletion(true) ? 0 : 1; }
From source file:com.jbw.taroutputformat.JobDriver.java
@Override public int run(String[] strings) throws Exception { Configuration conf = getConf(); Path in = new Path(conf.get("input")); Path out = new Path(conf.get("output")); Job job = Job.getInstance(); job.setJobName("test"); job.setInputFormatClass(FileInputFormat.class); job.setOutputFormatClass(TarOutputFormat.class); FileInputFormat.addInputPath(job, in); TarOutputFormat.setOutputPath(job, out); return job.waitForCompletion(true) ? 0 : 1; }