List of usage examples for org.apache.hadoop.mapreduce Job setJobName
public void setJobName(String name) throws IllegalStateException
From source file:eu.scape_project.tb.wc.archd.mapreduce.TikaCharacterisation.java
License:Apache License
public int run(String[] args) throws Exception { Configuration conf = getConf(); Job job = new Job(conf); System.out.println(conf.get("mapreduce.job.user.classpath.first")); for (int i = 0; i < args.length; i++) { System.out.println("Arg" + i + ": " + args[i]); }// w ww .ja v a 2s . c o m FileInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setJarByClass(TikaCharacterisation.class); job.setJobName(name); //*** Set interface data types // We are using LONG because this value can become very large on huge archives. // In order to use the combiner function, also the map output needs to be a LONG. //job.setMapOutputKeyClass(Text.class); //job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); //*** Set up the mapper, combiner and reducer job.setMapperClass(TikaMap.class); job.setCombinerClass(Reduce.class); job.setReducerClass(Reduce.class); //*** Set the MAP output compression //job.getConfiguration().set("mapred.compress.map.output", "true"); //*** Set input / output format job.setInputFormatClass(ArcInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); //*** Start the job and wait for it boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
From source file:example.MapReduceColorCount.java
License:Apache License
public int run(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: MapReduceColorCount <input path> <output path>"); return -1; }/*from w w w.j a va 2 s .c o m*/ Job job = new Job(getConf()); job.setJarByClass(MapReduceColorCount.class); job.setJobName("Color Count"); FileInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.setInputFormatClass(AvroKeyInputFormat.class); job.setMapperClass(ColorCountMapper.class); AvroJob.setInputKeySchema(job, User.getClassSchema()); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputFormatClass(AvroKeyValueOutputFormat.class); job.setReducerClass(ColorCountReducer.class); AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING)); AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT)); return (job.waitForCompletion(true) ? 0 : 1); }
From source file:fire.util.fileformats.combineimagefileinputformat.MultiImageFilesToSequenceFiles.java
License:Apache License
public int run(String[] args) throws Exception { if (args.length < 2) { printUsage();/* ww w. j ava 2 s. co m*/ return 2; } Job job = new Job(getConf()); job.setJobName("MultiImageFilesToSequenceFiles"); job.setJarByClass(MultiImageFilesToSequenceFiles.class); //set the InputFormat of the job to our InputFormat job.setInputFormatClass(CombineFileImageInputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); // the keys are words (strings) job.setOutputKeyClass(Text.class); // the values are images job.setOutputValueClass(BytesWritable.class); //use the defined mapper job.setMapperClass(MapClass.class); FileInputFormat.addInputPaths(job, args[0]); FileOutputFormat.setOutputPath(job, new Path(args[1])); return job.waitForCompletion(true) ? 0 : 1; }
From source file:fire.util.fileformats.combinetextfileinputformat.MultiFileWordCount.java
License:Apache License
public int run(String[] args) throws Exception { if (args.length < 2) { printUsage();/*w ww .j a va2 s. co m*/ return 2; } Job job = new Job(getConf()); job.setJobName("MultiFileWordCount"); job.setJarByClass(MultiFileWordCount.class); //set the InputFormat of the job to our InputFormat job.setInputFormatClass(CombineFileTextInputFormat.class); // the keys are words (strings) job.setOutputKeyClass(Text.class); // the values are counts (ints) job.setOutputValueClass(IntWritable.class); //use the defined mapper job.setMapperClass(MapClass.class); //use the WordCount Reducer job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); FileInputFormat.addInputPaths(job, args[0]); FileOutputFormat.setOutputPath(job, new Path(args[1])); return job.waitForCompletion(true) ? 0 : 1; }
From source file:fire.util.fileformats.tika.TikaDriver.java
License:Apache License
public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); @SuppressWarnings("deprecation") Job job = new Job(conf, "TikaParser"); job.setJarByClass(TikaDriver.class); job.setJobName("TikaDriver"); job.setInputFormatClass(TikaFileInputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); job.setMapperClass(TikaMapper.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(0);//from ww w .j a v a2s . c o m // job.setOutputFormatClass(TikaOutPutFormt.class); FileOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true); }
From source file:flink.applications.model.fraud.prepare.MarkovStateTransitionModel.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = new Job(getConf()); String jobName = "Markov tate transition model"; job.setJobName(jobName); job.setJarByClass(MarkovStateTransitionModel.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); Utility.setConfiguration(job.getConfiguration(), "avenir"); job.setMapperClass(StateTransitionMapper.class); job.setReducerClass(StateTransitionReducer.class); job.setCombinerClass(StateTransitionCombiner.class); job.setMapOutputKeyClass(Tuple.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(job.getConfiguration().getInt("num.reducer", 1)); int status = job.waitForCompletion(true) ? 0 : 1; return status; }
From source file:flink.applications.model.fraud.prepare.Projection.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job job = new Job(getConf()); String jobName = "Projection and grouping MR"; job.setJobName(jobName); job.setJarByClass(Projection.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); Utility.setConfiguration(job.getConfiguration()); String operation = job.getConfiguration().get("projection.operation", "project"); if (operation.startsWith("grouping")) { //group by job.setMapperClass(Projection.ProjectionMapper.class); job.setReducerClass(Projection.ProjectionReducer.class); job.setMapOutputKeyClass(Tuple.class); job.setMapOutputValueClass(Text.class); job.setNumReduceTasks(job.getConfiguration().getInt("num.reducer", 1)); //order by boolean doOrderBy = job.getConfiguration().getInt("orderBy.field", -1) >= 0; if (doOrderBy) { job.setGroupingComparatorClass(SecondarySort.TuplePairGroupComprator.class); job.setPartitionerClass(SecondarySort.TupleTextPartitioner.class); }//from w w w .j av a2s .c o m } else { //simple projection job.setMapperClass(Projection.SimpleProjectionMapper.class); } job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); int status = job.waitForCompletion(true) ? 0 : 1; return status; }
From source file:fm.last.darling.ZohmgProgram.java
License:Apache License
public int start(String input) throws Exception { Path path = new Path(input); // TODO: read table/dataset from environment. String table = "zohmg"; Job job = new Job(); job.setJobName("zohmg!"); FileInputFormat.addInputPath(job, path); Path output = new Path("yeah"); FileOutputFormat.setOutputPath(job, output); // input//from w w w . j a v a 2 s .c o m job.setInputFormatClass(TextInputFormat.class); // wrapper job.setMapperClass(MapperWrapper.class); job.setMapOutputKeyClass(NSpacePoint.class); job.setMapOutputValueClass(IntWritable.class); // output job.setCombinerClass(ZohmgCombiner.class); job.setReducerClass(ZohmgReducer.class); job.setOutputFormatClass(TableOutputFormat.class); job.setOutputKeyClass(ImmutableBytesWritable.class); job.setOutputValueClass(Put.class); //job.set(TableOutputFormat.OUTPUT_TABLE, table); return job.waitForCompletion(true) ? 0 : 1; }
From source file:full_MapReduce.C4_5.java
License:Open Source License
private static void summarizeData() throws Exception { Job job = Job.getInstance(); job.setJarByClass(C4_5.class); job.setJobName("C4.5_summarizeData"); FileInputFormat.addInputPath(job, input_path); FileOutputFormat.setOutputPath(job, summarized_data_path); job.setMapperClass(SummarizeMapper.class); job.setReducerClass(SummarizeReducer.class); job.setOutputKeyClass(TextArrayWritable.class); job.setOutputValueClass(IntWritable.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.waitForCompletion(false);/*from www . j a va2s . com*/ }
From source file:full_MapReduce.C4_5.java
License:Open Source License
private static void calcAttributesInfo(Map<String, String> conditions) throws Exception { Configuration conf = new Configuration(); for (Entry<String, String> condition : conditions.entrySet()) { conf.setStrings(condition.getKey(), condition.getValue()); }/*from w w w . ja v a 2 s. c om*/ Job job = Job.getInstance(conf); job.setJarByClass(C4_5.class); job.setJobName("C4.5_calcAttributesInfo"); FileInputFormat.addInputPath(job, summarized_data_path); FileOutputFormat.setOutputPath(job, calc_attributes_info_path); job.setMapperClass(AttributeInfoMapper.class); job.setReducerClass(AttributeInfoReducer.class); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(AttributeCounterWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(MapWritable.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.waitForCompletion(false); }