List of usage examples for org.apache.hadoop.mapreduce Job setOutputValueClass
public void setOutputValueClass(Class<?> theClass) throws IllegalStateException
From source file:accesslog4.Accesslog4.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path inputPath = new Path(args[0]); Path outputPath = new Path(args[1]); Job job = Job.getInstance(conf, "Accesslog4"); job.setJarByClass(Accesslog4.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, inputPath); FileOutputFormat.setOutputPath(job, outputPath); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:adts.ContainingArticle.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "ContainingArticle"); job.setJarByClass(ContainingArticle.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); Path queriesInputPath = new Path(args[0]); Path articlesInputPath = new Path(args[1]); MultipleInputs.addInputPath(job, queriesInputPath, TextInputFormat.class, QueriesMap.class); MultipleInputs.addInputPath(job, articlesInputPath, TextInputFormat.class, ArticlesMap.class); FileOutputFormat.setOutputPath(job, new Path("/root/temporary")); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.waitForCompletion(true);/*from w ww . ja v a 2 s . co m*/ Job collectingJob = new Job(conf, "ContainingArticle"); collectingJob.setJarByClass(ContainingArticle.class); collectingJob.setOutputKeyClass(IntWritable.class); collectingJob.setOutputValueClass(Text.class); collectingJob.setMapperClass(CollectMap.class); collectingJob.setReducerClass(CollectReduce.class); collectingJob.setInputFormatClass(TextInputFormat.class); collectingJob.setOutputFormatClass(TextOutputFormat.class); collectingJob.setMapOutputKeyClass(Text.class); collectingJob.setMapOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(collectingJob, new Path("/root/temporary")); FileOutputFormat.setOutputPath(collectingJob, new Path("/root/temporary2")); collectingJob.waitForCompletion(true); Job countingJob = new Job(conf, "ContainingArticle"); countingJob.setJarByClass(ContainingArticle.class); countingJob.setOutputKeyClass(Text.class); countingJob.setOutputValueClass(IntWritable.class); countingJob.setMapperClass(CountMap.class); countingJob.setReducerClass(CountReduce.class); countingJob.setInputFormatClass(TextInputFormat.class); countingJob.setOutputFormatClass(TextOutputFormat.class); countingJob.setMapOutputKeyClass(IntWritable.class); countingJob.setMapOutputValueClass(Text.class); FileInputFormat.addInputPath(countingJob, new Path("/root/temporary2")); FileOutputFormat.setOutputPath(countingJob, new Path(args[2])); countingJob.waitForCompletion(true); }
From source file:adts.PopularKeywords.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "PopularKeywords"); job.setJarByClass(PopularKeywords.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); Path queriesInputPath = new Path(args[0]); Path StopWordsInputPath = new Path(args[1]); MultipleInputs.addInputPath(job, queriesInputPath, TextInputFormat.class, Map.class); MultipleInputs.addInputPath(job, StopWordsInputPath, TextInputFormat.class, StopwordsMap.class); FileOutputFormat.setOutputPath(job, new Path("/root/temporary")); job.waitForCompletion(true);//from ww w.j a v a2 s .c om Job sortingJob = new Job(conf, "PopularKeywords"); sortingJob.setJarByClass(PopularKeywords.class); sortingJob.setOutputKeyClass(Text.class); sortingJob.setOutputValueClass(LongWritable.class); sortingJob.setMapperClass(ReverseMap.class); sortingJob.setReducerClass(ReverseReduce.class); sortingJob.setInputFormatClass(TextInputFormat.class); sortingJob.setOutputFormatClass(TextOutputFormat.class); sortingJob.setSortComparatorClass(LongWritable.DecreasingComparator.class); sortingJob.setMapOutputKeyClass(LongWritable.class); sortingJob.setMapOutputValueClass(Text.class); FileInputFormat.addInputPath(sortingJob, new Path("/root/temporary")); FileOutputFormat.setOutputPath(sortingJob, new Path(args[2])); sortingJob.setNumReduceTasks(1); sortingJob.waitForCompletion(true); }
From source file:adts.PrepareInput.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "PrepareInput"); job.setJarByClass(PrepareInput.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setNumReduceTasks(1);//from w ww.j a va 2 s.c o m job.waitForCompletion(true); }
From source file:adts.SuccessfullQueries.java
License:Open Source License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "SuccessfullQueries"); job.setJarByClass(SuccessfullQueries.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true);//from www .j a v a2 s . c om }
From source file:AllLab_Skeleton.Lab1.Lab1_Wordcount.java
/** * @param args the command line arguments *///from w w w .j a va 2 s .co m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(Lab1_Wordcount.class); job.setMapperClass(WordCount_Mapper.class); job.setCombinerClass(WordCount_Reducer.class); job.setReducerClass(WordCount_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:AllLab_Skeleton.Lab2.Lab2SecondarySort.java
/** * @param args the command line arguments *///from w w w . j av a 2 s . c om public static void main(String[] args) { try { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "SecondarySort"); job.setJarByClass(Lab2SecondarySort.class); job.setMapperClass(Lab2Mapper.class); job.setMapOutputKeyClass(CompositeKeyWritable.class); job.setMapOutputValueClass(NullWritable.class); job.setPartitionerClass(Lab2Partitioner.class); job.setGroupingComparatorClass(Lab2GroupComparator.class); job.setReducerClass(Lab2Reducer.class); job.setOutputKeyClass(CompositeKeyWritable.class); job.setOutputValueClass(NullWritable.class); job.setNumReduceTasks(8); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); } catch (IOException | InterruptedException | ClassNotFoundException ex) { System.out.println("Erorr Message" + ex.getMessage()); } }
From source file:AllLab_Skeleton.Lab4.Lab4_Std_dev.java
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "medianstd"); job.setJarByClass(Lab4_Std_dev.class); job.setMapperClass(Map.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(DoubleWritable.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(MedianSDCustomWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:AllLab_Skeleton.Lab6.ReduceSideJoin.java
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "ReduceSideJoin"); job.setJarByClass(ReduceSideJoin.class); // Use MultipleInputs to set which input uses what mapper // This will keep parsing of each data set separate from a logical // standpoint // The first two elements of the args array are the two inputs MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, UserJoinMapper.class); MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, CommentJoinMapper.class); job.getConfiguration().set("join.type", "leftouter"); //job.setNumReduceTasks(0); job.setReducerClass(UserJoinReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, new Path(args[2])); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.waitForCompletion(true);/*w w w. j a v a 2 s .com*/ }
From source file:alluxio.checker.MapReduceIntegrationChecker.java
License:Apache License
/** * Implements MapReduce with Alluxio integration checker. * * @return 0 for success, 2 for unable to find Alluxio classes, 1 otherwise *///from w w w . j a v a 2 s. co m private int run(String[] args) throws Exception { Configuration conf = new Configuration(); String numMaps = new GenericOptionsParser(conf, args).getRemainingArgs()[0]; conf.set(MRJobConfig.NUM_MAPS, numMaps); createHdfsFilesystem(conf); Job job = Job.getInstance(conf, "MapReduceIntegrationChecker"); job.setJarByClass(MapReduceIntegrationChecker.class); job.setMapperClass(CheckerMapper.class); job.setCombinerClass(CheckerReducer.class); job.setReducerClass(CheckerReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setInputFormatClass(EmptyInputFormat.class); FileOutputFormat.setOutputPath(job, mOutputFilePath); try { if (!job.waitForCompletion(true)) { return 1; } Status resultStatus = generateReport(); return resultStatus.equals(Status.SUCCESS) ? 0 : (resultStatus.equals(Status.FAIL_TO_FIND_CLASS) ? 2 : 1); } finally { if (mFileSystem.exists(mOutputFilePath)) { mFileSystem.delete(mOutputFilePath, true); } mFileSystem.close(); } }