List of usage examples for org.apache.hadoop.mapreduce Job setOutputFormatClass
public void setOutputFormatClass(Class<? extends OutputFormat> cls) throws IllegalStateException
From source file:BigramRelativeFrequencyTuple.java
License:Apache License
/** * Runs this tool.//from w ww .ja va 2 s . com */ public int run(String[] args) throws Exception { if (args.length != 3) { printUsage(); return -1; } String inputPath = args[0]; String outputPath = args[1]; int reduceTasks = Integer.parseInt(args[2]); LOG.info("Tool name: " + BigramRelativeFrequencyTuple.class.getSimpleName()); LOG.info(" - input path: " + inputPath); LOG.info(" - output path: " + outputPath); LOG.info(" - num reducers: " + reduceTasks); Job job = Job.getInstance(getConf()); job.setJobName(BigramRelativeFrequencyTuple.class.getSimpleName()); job.setJarByClass(BigramRelativeFrequencyTuple.class); job.setNumReduceTasks(reduceTasks); FileInputFormat.setInputPaths(job, new Path(inputPath)); FileOutputFormat.setOutputPath(job, new Path(outputPath)); job.setMapOutputKeyClass(BinSedesTuple.class); job.setMapOutputValueClass(FloatWritable.class); job.setOutputKeyClass(BinSedesTuple.class); job.setOutputValueClass(FloatWritable.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setMapperClass(MyMapper.class); job.setCombinerClass(MyCombiner.class); job.setReducerClass(MyReducer.class); job.setPartitionerClass(MyPartitioner.class); // Delete the output directory if it exists already. Path outputDir = new Path(outputPath); FileSystem.get(getConf()).delete(outputDir, true); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); System.out.println("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); return 0; }
From source file:accesslog2.Accesslog2.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Job job = Job.getInstance(conf, "Accesslog2"); job.setJarByClass(Accesslog2.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:accesslog3.Accesslog3.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Job job = Job.getInstance(conf, "Accesslog3"); job.setJarByClass(Accesslog3.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:accesslog4.Accesslog4.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Job job = Job.getInstance(conf, "Accesslog4"); job.setJarByClass(Accesslog4.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:accismus.benchmark.Generator.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 3) { System.err.println(/*from w w w . j a va2 s .c om*/ "Usage : " + this.getClass().getSimpleName() + " <props file> <num task> <num documents>"); return 1; } Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); job.setInputFormatClass(DocumentInputFormat.class); DocumentInputFormat.setNumTask(job, Integer.parseInt(args[1]), Integer.parseInt(args[2])); job.setMapOutputKeyClass(Loader.class); job.setMapOutputValueClass(NullWritable.class); job.setMapperClass(GMapper.class); job.setNumReduceTasks(0); job.setOutputFormatClass(AccismusOutputFormat.class); Properties accisumusProps = new Properties(); accisumusProps.load(new FileReader(args[0])); AccismusOutputFormat.configure(job, accisumusProps); job.waitForCompletion(true); return 0; }
From source file:accismus.benchmark.Verifier.java
License:Apache License
@Override public int run(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage : " + this.getClass().getSimpleName() + " <props file> <output dir>"); return 1; }/*from w w w .ja v a 2s . c om*/ Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); job.setJarByClass(this.getClass()); AccismusProperties accisumusProps = new AccismusProperties(new File(args[0])); AccismusInputFormat.configure(job, accisumusProps); AccismusInputFormat.fetchFamilies(job, KEY, DUP); job.setInputFormatClass(AccismusInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setMapperClass(VMapper.class); job.setReducerClass(VReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true); return 0; }
From source file:adts.ContainingArticle.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "ContainingArticle"); job.setJarByClass(ContainingArticle.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); Path queriesInputPath = new Path(args[0]); Path articlesInputPath = new Path(args[1]); MultipleInputs.addInputPath(job, queriesInputPath, TextInputFormat.class, QueriesMap.class); MultipleInputs.addInputPath(job, articlesInputPath, TextInputFormat.class, ArticlesMap.class); FileOutputFormat.setOutputPath(job, new Path("/root/temporary")); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.waitForCompletion(true);//from www. ja va 2 s. c o m Job collectingJob = new Job(conf, "ContainingArticle"); collectingJob.setJarByClass(ContainingArticle.class); collectingJob.setOutputKeyClass(IntWritable.class); collectingJob.setOutputValueClass(Text.class); collectingJob.setMapperClass(CollectMap.class); collectingJob.setReducerClass(CollectReduce.class); collectingJob.setInputFormatClass(TextInputFormat.class); collectingJob.setOutputFormatClass(TextOutputFormat.class); collectingJob.setMapOutputKeyClass(Text.class); collectingJob.setMapOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(collectingJob, new Path("/root/temporary")); FileOutputFormat.setOutputPath(collectingJob, new Path("/root/temporary2")); collectingJob.waitForCompletion(true); Job countingJob = new Job(conf, "ContainingArticle"); countingJob.setJarByClass(ContainingArticle.class); countingJob.setOutputKeyClass(Text.class); countingJob.setOutputValueClass(IntWritable.class); countingJob.setMapperClass(CountMap.class); countingJob.setReducerClass(CountReduce.class); countingJob.setInputFormatClass(TextInputFormat.class); countingJob.setOutputFormatClass(TextOutputFormat.class); countingJob.setMapOutputKeyClass(IntWritable.class); countingJob.setMapOutputValueClass(Text.class); FileInputFormat.addInputPath(countingJob, new Path("/root/temporary2")); FileOutputFormat.setOutputPath(countingJob, new Path(args[2])); countingJob.waitForCompletion(true); }
From source file:adts.PopularKeywords.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "PopularKeywords"); job.setJarByClass(PopularKeywords.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); Path queriesInputPath = new Path(args[0]); Path StopWordsInputPath = new Path(args[1]); MultipleInputs.addInputPath(job, queriesInputPath, TextInputFormat.class, Map.class); MultipleInputs.addInputPath(job, StopWordsInputPath, TextInputFormat.class, StopwordsMap.class); FileOutputFormat.setOutputPath(job, new Path("/root/temporary")); job.waitForCompletion(true);/*from w w w. jav a2 s . c o m*/ Job sortingJob = new Job(conf, "PopularKeywords"); sortingJob.setJarByClass(PopularKeywords.class); sortingJob.setOutputKeyClass(Text.class); sortingJob.setOutputValueClass(LongWritable.class); sortingJob.setMapperClass(ReverseMap.class); sortingJob.setReducerClass(ReverseReduce.class); sortingJob.setInputFormatClass(TextInputFormat.class); sortingJob.setOutputFormatClass(TextOutputFormat.class); sortingJob.setSortComparatorClass(LongWritable.DecreasingComparator.class); sortingJob.setMapOutputKeyClass(LongWritable.class); sortingJob.setMapOutputValueClass(Text.class); FileInputFormat.addInputPath(sortingJob, new Path("/root/temporary")); FileOutputFormat.setOutputPath(sortingJob, new Path(args[2])); sortingJob.setNumReduceTasks(1); sortingJob.waitForCompletion(true); }
From source file:adts.PrepareInput.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "PrepareInput"); job.setJarByClass(PrepareInput.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setNumReduceTasks(1);/*from w w w. j av a2 s . c o m*/ job.waitForCompletion(true); }
From source file:adts.SuccessfullQueries.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "SuccessfullQueries"); job.setJarByClass(SuccessfullQueries.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true);/* w ww . j a v a 2 s .c om*/ }