List of usage examples for org.apache.hadoop.mapreduce Job setOutputKeyClass
public void setOutputKeyClass(Class<?> theClass) throws IllegalStateException
From source file:arpserver.HadoopTool.java
@Override public int run(String[] strings) throws Exception { Configuration conf = new Configuration(); String in = strings[0];//from ww w.jav a2s.co m String out = strings[1]; FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(out))) { fs.delete(new Path(out), true); fs.delete(new Path(out + "Src"), true); fs.delete(new Path(out + "Mitm"), true); fs.delete(new Path(out + "ArpScn"), true); fs.delete(new Path(out + "s"), true); fs.delete(new Path(out + "d"), true); fs.delete(new Path(out + "t"), true); } Job job = new Job(); Job job2 = new Job(); Job job3 = new Job(); Job job4 = new Job(); Job job5 = new Job(); Job job6 = new Job(); Job job7 = new Job(); job.setJobName("Q"); job2.setJobName("Src"); job3.setJobName("Mitm"); job4.setJobName("ArpScn"); job5.setJobName("s"); job6.setJobName("d"); job7.setJobName("time"); job.setJarByClass(QuickDetect.class); job.setMapperClass(Qmapper.class); job.setReducerClass(Qreducer.class); job2.setMapperClass(Srcmapper.class); job2.setReducerClass(Srcreducer.class); job3.setMapperClass(ArpScanmapper.class); job3.setReducerClass(ArpScanreducer.class); job4.setMapperClass(Mitmmapper.class); job4.setReducerClass(Mitmreducer.class); job5.setMapperClass(Smapper.class); job5.setReducerClass(Sreducer.class); job6.setMapperClass(Dmapper.class); job6.setReducerClass(Dreducer.class); job7.setMapperClass(timemapper.class); job7.setReducerClass(timereducer.class); //testFinal168.txt job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job2.setOutputKeyClass(NullWritable.class); job2.setOutputValueClass(Text.class); job3.setOutputKeyClass(NullWritable.class); job3.setOutputValueClass(IntWritable.class); job4.setOutputKeyClass(NullWritable.class); job4.setOutputValueClass(Text.class); job5.setOutputKeyClass(NullWritable.class); job5.setOutputValueClass(Text.class); job6.setOutputKeyClass(NullWritable.class); job6.setOutputValueClass(Text.class); job7.setOutputKeyClass(NullWritable.class); job7.setOutputValueClass(Text.class); job.setMapOutputKeyClass(QuickDetect.class); job.setMapOutputValueClass(IntWritable.class); //job.setOutputFormatClass(YearMultipleTextOutputFormat.class); job2.setMapOutputKeyClass(DetectSrc.class); job2.setMapOutputValueClass(IntWritable.class); job3.setMapOutputKeyClass(DetectArpScan.class); job3.setMapOutputValueClass(IntWritable.class); job4.setMapOutputKeyClass(DetectMitm.class); job4.setMapOutputValueClass(IntWritable.class); job5.setMapOutputKeyClass(SMac.class); job5.setMapOutputValueClass(IntWritable.class); job6.setMapOutputKeyClass(DMac.class); job6.setMapOutputValueClass(IntWritable.class); job7.setMapOutputKeyClass(timeMac.class); job7.setMapOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(in)); FileOutputFormat.setOutputPath(job, new Path(out)); if (job.waitForCompletion(true)) { FileInputFormat.addInputPath(job2, new Path(in)); FileOutputFormat.setOutputPath(job2, new Path(out + "Src")); if (job2.waitForCompletion(true)) { FileInputFormat.addInputPath(job3, new Path(in)); FileOutputFormat.setOutputPath(job3, new Path(out + "ArpScn")); if (job3.waitForCompletion(true)) { FileInputFormat.addInputPath(job4, new Path(in)); FileOutputFormat.setOutputPath(job4, new Path(out + "Mitm")); if (job4.waitForCompletion(true)) { FileInputFormat.addInputPath(job5, new Path(in)); FileOutputFormat.setOutputPath(job5, new Path(out + "s")); if (job5.waitForCompletion(true)) { FileInputFormat.addInputPath(job6, new Path(in)); FileOutputFormat.setOutputPath(job6, new Path(out + "d")); if (job6.waitForCompletion(true)) { FileInputFormat.addInputPath(job7, new Path(in)); FileOutputFormat.setOutputPath(job7, new Path(out + "t")); job7.waitForCompletion(true); } else { return 1; } } else { return 1; } } else { return 1; } } else { return 1; } } else { return 1; } } else { return 1; } return 0; }
From source file:AshleyIngram.FYP.Hadoop.WordCount.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: wordcount <in> <out>"); System.exit(2);// ww w .j a v a 2 s . com } Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:assignment1.WordCount.LinkedSort.LinkedSort.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { System.err.println("Usage: hadoop jar This.jar <in> [<in>...] <out>"); System.exit(2);/* w w w .ja v a2s . co m*/ } Job job = new Job(conf, "word count"); job.setJarByClass(LinkedSort.class); job.setMapperClass(TokenizerMapper.class); //job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setPartitionerClass(SortPartitioner.class); job.setOutputKeyClass(WordAndLength.class); job.setOutputValueClass(IntWritable.class); job.setNumReduceTasks(2); for (int i = 0; i < otherArgs.length - 1; ++i) { FileInputFormat.addInputPath(job, new Path(otherArgs[i])); } FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:assignment1.WordCount.WordCountInMap.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { System.err.println("Usage: hadoop jar This.jar <in> [<in>...] <out>"); System.exit(2);//w w w.ja v a 2 s . c o m } Job job = new Job(conf, "word count"); job.setJarByClass(assignment1.WordCount.WordCountInMap.class); job.setMapperClass(TokenizerMapper.class); //job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); for (int i = 0; i < otherArgs.length - 1; ++i) { FileInputFormat.addInputPath(job, new Path(otherArgs[i])); } FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:assignment1.WordCount.WordSort.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { System.err.println("Usage: hadoop jar This.jar <in> [<in>...] <out>"); System.exit(2);/*w w w . j a va 2 s . c o m*/ } Job job = new Job(conf, "word count"); job.setJarByClass(WordSort.class); job.setMapperClass(TokenizerMapper.class); //job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setPartitionerClass(SortPartitioner.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setNumReduceTasks(2); for (int i = 0; i < otherArgs.length - 1; ++i) { FileInputFormat.addInputPath(job, new Path(otherArgs[i])); } FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P2_StockExchangeCount.StockPriceDriver.java
/** * @param args the command line arguments *///from w ww .j a v a 2 s.c om public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(StockPriceDriver.class); job.setMapperClass(StockPrice_Mapper.class); job.setCombinerClass(StockPrice_Reducer.class); job.setReducerClass(StockPrice_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FloatWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P3_GenderMovieCount.GenderMovieRatingDriver.java
/** * @param args the command line arguments *///from w ww .ja va 2 s . c o m public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Gender Movie Rating Count"); job.setJarByClass(GenderMovieRatingDriver.class); job.setMapperClass(GenderMovieRating_Mapper.class); job.setCombinerClass(GenderMovieRating_Reducer.class); job.setReducerClass(GenderMovieRating_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P4_MovieRatingCount.MovieRatingDriver.java
/** * @param args the command line arguments *//*from w w w .ja va 2 s . c o m*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "Movie Rating Count"); job.setJarByClass(MovieRatingDriver.class); job.setMapperClass(MovieRating_Mapper.class); job.setCombinerClass(MovieRating_Reducer.class); job.setReducerClass(MovieRating_Reducer.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment2_P5_IPAddressCount.IPAddressDriver.java
/** * @param args the command line arguments */// w w w . ja v a 2s . com public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "IP Address Count"); job.setJarByClass(IPAddressDriver.class); job.setMapperClass(IPAddress_Mapper.class); job.setCombinerClass(IPAddress_Reducer.class); job.setReducerClass(IPAddress_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:Assignment3_P2_MergeStockAverageCount.StockPriceMergeDriver.java
/** * @param args the command line arguments *//*www.ja v a 2 s .c om*/ public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); // local file system handle FileSystem local = FileSystem.getLocal(conf); // hdfs file system handle FileSystem hdfs = FileSystem.get(conf); // local input directory Path inputDir = new Path(args[0]); // hdfs i/p directory Path inputDir1 = new Path(args[1]); // local input files in local dir FileStatus[] inputFiles = local.listStatus(inputDir); // o/p stream FSDataOutputStream out = hdfs.create(inputDir1); // open each file and extract contents of file for (int i = 0; i < inputFiles.length; i++) { System.out.println("File name ----------------------------------------------------------------> " + inputFiles[i].getPath().getName()); FSDataInputStream in = local.open(inputFiles[i].getPath()); byte buffer[] = new byte[256]; int bytesRead = 0; // extract all contents of file while ((bytesRead = in.read(buffer)) > 0) { out.write(buffer, 0, bytesRead); } // close input stream in.close(); } Job job = Job.getInstance(conf, "Average Stock Price"); job.setJarByClass(StockPriceMergeDriver.class); job.setMapperClass(StockPriceMerge_Mapper.class); job.setCombinerClass(StockPriceMerge_Reducer.class); job.setReducerClass(StockPriceMerge_Reducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(FloatWritable.class); FileInputFormat.addInputPath(job, new Path(args[1])); // above programs output will be input for mapper FileOutputFormat.setOutputPath(job, new Path(args[2])); System.exit(job.waitForCompletion(true) ? 0 : 1); }