List of usage examples for org.apache.hadoop.mapreduce.lib.jobcontrol JobControl getFailedJobList
synchronized public List<ControlledJob> getFailedJobList()
From source file:com.laizuozuoba.WordCount.java
License:Apache License
public static void main(String[] args) throws Exception { // System.setProperty("hadoop.home.dir", "D:\\hadoop-2.2.0"); Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: wordcount <in> <out>"); System.exit(2);//from w w w . j a v a2 s. co m } Job job = new Job(conf, "word count"); job.setJarByClass(WordCount.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); Job job2 = new Job(conf, "uv"); job2.setJarByClass(WordCount.class); job2.setMapperClass(UVMapper.class); job2.setCombinerClass(UVReducer.class); job2.setReducerClass(UVReducer.class); job2.setOutputKeyClass(Text.class); job2.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job2, new Path(otherArgs[1])); FileOutputFormat.setOutputPath(job2, new Path("hdfs://10.18.106.67:9100/result2")); ControlledJob controlledJob = new ControlledJob(job.getConfiguration()); ControlledJob controlledJob2 = new ControlledJob(job2.getConfiguration()); controlledJob2.addDependingJob(controlledJob); JobControl jc = new JobControl("123"); jc.addJob(controlledJob); jc.addJob(controlledJob2); Thread jcThread = new Thread(jc); jcThread.start(); while (true) { if (jc.allFinished()) { System.out.println(jc.getSuccessfulJobList()); jc.stop(); break; } if (jc.getFailedJobList().size() > 0) { System.out.println(jc.getFailedJobList()); jc.stop(); break; } Thread.sleep(1000); } System.out.println("Finished!!!!!!!!!!!!!!!!!!!!!!!"); }
From source file:tv.icntv.log.stb.GenerateStbLogJob.java
License:Apache License
@Override public boolean run(Map<String, String> maps) throws Exception { //To change body of implemented methods use File | Settings | File Templates. Configuration configuration = getConf(); //user login//from w w w . j a v a2 s .com Job userLogin = Job.getInstance(configuration, "user login job"); userLogin.setMapperClass(ParserLoginMapper.class); userLogin.setJarByClass(this.getClass()); userLogin.setOutputKeyClass(NullWritable.class); userLogin.setOutputValueClass(Text.class); Path userLoginOutput = new Path(maps.get(USER_LOGIN_JOB_OUTPUT)); FileInputFormat.addInputPath(userLogin, new Path(maps.get(USER_LOGIN_JOB_INPUT))); FileOutputFormat.setOutputPath(userLogin, userLoginOutput); userLogin.setNumReduceTasks(0); ControlledJob userControlledJob = new ControlledJob(configuration); userControlledJob.setJob(userLogin); // //player Job player = Job.getInstance(configuration, "player job"); player.setJarByClass(getClass()); player.setMapperClass(PlayerMapper.class); player.setOutputValueClass(Text.class); player.setOutputKeyClass(NullWritable.class); FileInputFormat.addInputPath(player, new Path(maps.get(PLAYER_JOB_INPUT))); FileOutputFormat.setOutputPath(player, new Path(maps.get(PLAYER_JOB_OUTPUT))); player.setNumReduceTasks(0); ControlledJob playControlledJob = new ControlledJob(configuration); playControlledJob.setJob(player); //contentView Job contentView = Job.getInstance(configuration, "content view job"); contentView.setJarByClass(getClass()); contentView.setMapperClass(ContentViewMapperBack.class); contentView.setOutputKeyClass(NullWritable.class); contentView.setOutputValueClass(Text.class); //System.out.println(MessageFormat.format(maps.get(OUTPUT_PREFIX),day)+MessageFormat.format(maps.get(CONTENT_VIEW_JOB_INPUT),configuration.getLong(FILTER_TIME,0L))); FileInputFormat.addInputPath(contentView, new Path(maps.get(CONTENT_VIEW_JOB_INPUT))); FileOutputFormat.setOutputPath(contentView, new Path(maps.get(CONTENT_VIEW_JOB_OUTPUT))); contentView.setNumReduceTasks(0); ControlledJob contentViewControlledJob = new ControlledJob(configuration); contentViewControlledJob.setJob(contentView); //reply Job replay = Job.getInstance(configuration, "reply job"); replay.setJarByClass(getClass()); replay.setMapperClass(ReplayMapper.class); replay.setOutputKeyClass(NullWritable.class); replay.setOutputValueClass(Text.class); FileInputFormat.addInputPath(replay, new Path(maps.get(LOOK_BACK_JOB_INPUT))); FileOutputFormat.setOutputPath(replay, new Path(maps.get(LOOK_BACK_JOB_OUTPUT))); replay.setNumReduceTasks(0); ControlledJob replayControlledJob = new ControlledJob(configuration); replayControlledJob.setJob(replay); //logEpg Job logEpg = Job.getInstance(configuration, "log epg job"); logEpg.setJarByClass(getClass()); logEpg.setMapperClass(EPGOperateMapper.class); logEpg.setOutputKeyClass(NullWritable.class); logEpg.setOutputValueClass(Text.class); FileInputFormat.addInputPath(logEpg, new Path(maps.get(LOG_EPG_JOB_INPUT))); FileOutputFormat.setOutputPath(logEpg, new Path(maps.get(LOG_EPG_JOB_OUTPUT))); logEpg.setNumReduceTasks(0); ControlledJob logEpgControlledJob = new ControlledJob(configuration); logEpgControlledJob.setJob(logEpg); // //cdn // Job cdn=Job.getInstance(configuration,"cdn job"); // cdn.setJarByClass(getClass()); // cdn.setMapperClass(CdnModuleMapper.class); // cdn.setOutputKeyClass(NullWritable.class); // cdn.setOutputValueClass(Text.class); // FileInputFormat.addInputPath(cdn, new Path(maps.get(CDN_JOB_INPUT))); // FileOutputFormat.setOutputPath(cdn, new Path(maps.get(CDN_JOB_OUTPUT))); // cdn.setNumReduceTasks(0); // ControlledJob cdnControlledJob=new ControlledJob(configuration); // cdnControlledJob.setJob(cdn); //cdn stb Job cdn = Job.getInstance(configuration, "cdn stb job"); cdn.setJarByClass(this.getClass()); cdn.setMapperClass(CdnStbMapper.class); cdn.setOutputValueClass(Text.class); cdn.setOutputKeyClass(NullWritable.class); FileInputFormat.addInputPath(cdn, new Path(maps.get(CDN_JOB_INPUT))); FileOutputFormat.setOutputPath(cdn, new Path(maps.get(CDN_JOB_OUTPUT))); cdn.setNumReduceTasks(0); ControlledJob cdnControlledJob = new ControlledJob(configuration); cdnControlledJob.setJob(cdn); //cdn adapter Job cdnAdapterJob = Job.getInstance(configuration, "cdn adapter job "); cdnAdapterJob.setJarByClass(getClass()); cdnAdapterJob.setMapperClass(CdnAdapterMapper.class); cdnAdapterJob.setOutputKeyClass(NullWritable.class); cdnAdapterJob.setOutputValueClass(Text.class); FileInputFormat.addInputPath(cdnAdapterJob, new Path(maps.get(CDN_ADAPTER_JOB_INPUT))); FileOutputFormat.setOutputPath(cdnAdapterJob, new Path(maps.get(CDN_ADAPTER_JOB_OUTPUT))); cdnAdapterJob.setNumReduceTasks(0); ControlledJob cdnAdapterControlleredJob = new ControlledJob(configuration); cdnAdapterControlleredJob.setJob(cdnAdapterJob); JobControl jobControl = new JobControl("stb log parser .eg: userLogin,devicePlayer,contentView,logEpg,cdn"); jobControl.addJob(userControlledJob); jobControl.addJob(playControlledJob); jobControl.addJob(contentViewControlledJob); jobControl.addJob(replayControlledJob); jobControl.addJob(logEpgControlledJob); jobControl.addJob(cdnControlledJob); jobControl.addJob(cdnAdapterControlleredJob); new Thread(jobControl).start(); while (!jobControl.allFinished()) { Thread.sleep(5000); } if (jobControl.getFailedJobList().size() > 0) { return false; } return true; }