List of usage examples for org.apache.hadoop.mapreduce.lib.jobcontrol JobControl JobControl
public JobControl(String groupName)
From source file:com.zinnia.nectar.regression.hadoop.primitive.jobs.SigmaSqJob.java
License:Apache License
public Double call() throws NectarException { // TODO Auto-generated method stub double value = 0; JobControl jobControl = new JobControl("sigmajob"); try {// w w w .j a va 2 s . co m job = new Job(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } job.setJarByClass(SigmaSqJob.class); log.info("Sigma square Job initialized"); log.warn("Sigma square job: Processing...Do not terminate/close"); log.debug("Sigma square job: Mapping process started"); try { ChainMapper.addMapper(job, FieldSeperator.FieldSeperationMapper.class, DoubleWritable.class, Text.class, NullWritable.class, Text.class, job.getConfiguration()); ChainMapper.addMapper(job, SigmaSqMapper.class, NullWritable.class, Text.class, Text.class, DoubleWritable.class, job.getConfiguration()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } job.getConfiguration().set("fields.spec", "" + column); job.setReducerClass(DoubleSumReducer.class); try { FileInputFormat.addInputPath(job, new Path(inputFilePath)); fs = FileSystem.get(job.getConfiguration()); if (!fs.exists(new Path(inputFilePath))) { throw new NectarException("Exception occured:File " + inputFilePath + " not found "); } } catch (Exception e) { // TODO Auto-generated catch block String trace = new String(); log.error(e.toString()); for (StackTraceElement s : e.getStackTrace()) { trace += "\n\t at " + s.toString(); } log.debug(trace); log.debug("Sigma square Job terminated abruptly\n"); throw new NectarException(); } FileOutputFormat.setOutputPath(job, new Path(outputFilePath)); job.setOutputKeyClass(Text.class); job.setOutputValueClass(DoubleWritable.class); job.setInputFormatClass(TextInputFormat.class); log.debug("Sigma square job: Mapping process completed"); log.debug("Sigma square job: Reducing process started"); try { controlledJob = new ControlledJob(job.getConfiguration()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } jobControl.addJob(controlledJob); Thread thread = new Thread(jobControl); thread.start(); while (!jobControl.allFinished()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } jobControl.stop(); try { fs = FileSystem.get(job.getConfiguration()); FSDataInputStream in = fs.open(new Path(outputFilePath + "/part-r-00000")); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in)); String valueLine = bufferedReader.readLine(); String[] fields = valueLine.split("\t"); value = Double.parseDouble(fields[1]); bufferedReader.close(); in.close(); } catch (IOException e) { log.error("Exception occured: Output file cannot be read."); log.debug(e.getMessage()); log.debug("Sigma square Job terminated abruptly\n"); throw new NectarException(); } log.debug("Sigma square job: Reducing process completed"); log.info("Sigma square Job completed\n"); return value; }
From source file:com.zinnia.nectar.regression.hadoop.primitive.jobs.SigmaXYJob.java
License:Apache License
public Double call() throws NectarException { double value = 0; JobControl jobControl = new JobControl("sigmajob"); try {//from ww w . j a va 2 s . c o m job = new Job(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } job.setJarByClass(SigmaXYJob.class); log.info("SigmaXY Job initialized"); log.warn("SigmaXY job: Processing...Do not terminate/close"); log.debug("SigmaXY job: Mapping process started"); try { ChainMapper.addMapper(job, FieldSeperator.FieldSeperationMapper.class, LongWritable.class, Text.class, NullWritable.class, Text.class, job.getConfiguration()); ChainMapper.addMapper(job, SigmaXYMapper.class, NullWritable.class, Text.class, Text.class, DoubleWritable.class, job.getConfiguration()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } job.getConfiguration().set("fields.spec", x + "," + y); job.setReducerClass(DoubleSumReducer.class); try { FileInputFormat.addInputPath(job, new Path(inputFilePath)); fs = FileSystem.get(job.getConfiguration()); if (!fs.exists(new Path(inputFilePath))) { throw new NectarException("Exception occured:File " + inputFilePath + " not found "); } } catch (Exception e) { // TODO Auto-generated catch block String trace = new String(); log.error(e.toString()); for (StackTraceElement s : e.getStackTrace()) { trace += "\n\t at " + s.toString(); } log.debug(trace); log.debug("SigmaXY Job terminated abruptly\n"); throw new NectarException(); } FileOutputFormat.setOutputPath(job, new Path(outputFilePath)); job.setOutputKeyClass(Text.class); job.setOutputValueClass(DoubleWritable.class); job.setInputFormatClass(TextInputFormat.class); log.debug("SigmaXY job: Mapping process completed"); log.debug("SigmaXY job: Reducing process started"); try { controlledJob = new ControlledJob(job.getConfiguration()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } jobControl.addJob(controlledJob); Thread thread = new Thread(jobControl); thread.start(); while (!jobControl.allFinished()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } jobControl.stop(); FileSystem fs; try { fs = FileSystem.get(job.getConfiguration()); FSDataInputStream in = fs.open(new Path(outputFilePath + "/part-r-00000")); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in)); String valueLine = bufferedReader.readLine(); String[] fields = valueLine.split("\t"); value = Double.parseDouble(fields[1]); bufferedReader.close(); in.close(); } catch (IOException e) { log.error("Exception occured: Output file cannot be read."); log.debug(e.getMessage()); log.debug("SigmaXY Job terminated abruptly\n"); throw new NectarException(); } log.debug("SigmaXY job: Reducing process completed"); log.info("SigmaXY Job completed\n"); return value; }
From source file:com.zinnia.nectar.regression.hadoop.primitive.jobs.SortJob.java
License:Apache License
public Double[] call() throws NectarException { // TODO Auto-generated method stub JobControl jobControl = new JobControl("Sortjob"); try {// w ww .j a v a2s .c o m job = new Job(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } job.setJarByClass(SortJob.class); log.info("Sorting Job initialized"); log.warn("Sorting job: Processing...Do not terminate/close"); log.debug("Sorting job: Mapping process started"); try { ChainMapper.addMapper(job, FieldSeperator.FieldSeperationMapper.class, LongWritable.class, Text.class, NullWritable.class, Text.class, job.getConfiguration()); ChainMapper.addMapper(job, SortMapper.class, NullWritable.class, Text.class, DoubleWritable.class, DoubleWritable.class, job.getConfiguration()); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } job.getConfiguration().set("fields.spec", "" + column); job.setReducerClass(Reducer.class); try { FileInputFormat.addInputPath(job, new Path(inputFilePath)); fs = FileSystem.get(job.getConfiguration()); if (!fs.exists(new Path(inputFilePath))) { throw new NectarException("Exception occured:File " + inputFilePath + " not found "); } } catch (Exception e2) { // TODO Auto-generated catch block String trace = new String(); log.error(e2.toString()); for (StackTraceElement s : e2.getStackTrace()) { trace += "\n\t at " + s.toString(); } log.debug(trace); log.debug("Sorting Job terminated abruptly\n"); throw new NectarException(); } FileOutputFormat.setOutputPath(job, new Path(outputFilePath)); job.setMapOutputValueClass(DoubleWritable.class); job.setMapOutputKeyClass(DoubleWritable.class); job.setInputFormatClass(TextInputFormat.class); log.debug("Sorting job: Mapping process completed"); log.debug("Sorting job: Reducing process started"); try { controlledJob = new ControlledJob(job.getConfiguration()); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } jobControl.addJob(controlledJob); Thread thread = new Thread(jobControl); thread.start(); while (!jobControl.allFinished()) { try { Thread.sleep(10000); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } } try { FSDataInputStream in = fs.open(new Path(outputFilePath + "/part-r-00000")); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in)); String valueLine; while ((valueLine = bufferedReader.readLine()) != null) { String[] fields = valueLine.split("\t"); value.add(Double.parseDouble(fields[1])); } bufferedReader.close(); in.close(); } catch (IOException e) { // TODO Auto-generated catch block log.error("Exception occured: Output file cannot be read."); log.debug(e.getMessage()); log.debug("Sorting Job terminated abruptly\n"); throw new NectarException(); } log.debug("Sorting job: Reducing process completed"); log.info("Sorting Job completed\n"); return value.toArray(new Double[value.size()]); }
From source file:com.zinnia.nectar.regression.hadoop.primitive.jobs.YDiffJob.java
License:Apache License
public Double call() throws Exception { JobControl jobControl = new JobControl("YDiff job"); Job job = new Job(); job.setJarByClass(YDiffJob.class); ChainMapper.addMapper(job, FieldSeperator.FieldSeperationMapper.class, DoubleWritable.class, Text.class, NullWritable.class, Text.class, job.getConfiguration()); ChainMapper.addMapper(job, YDiffMapper.class, NullWritable.class, Text.class, Text.class, DoubleWritable.class, job.getConfiguration()); String fieldSpec = getFieldSpecForColumns(); job.getConfiguration().set("fields.spec", fieldSpec); job.getConfiguration().setStrings("paramValues", paramValues); job.setReducerClass(DoubleSumReducer.class); FileInputFormat.addInputPath(job, new Path(inputFilePath)); FileOutputFormat.setOutputPath(job, new Path(outputFilePath)); job.setOutputKeyClass(Text.class); job.setOutputValueClass(DoubleWritable.class); job.setInputFormatClass(TextInputFormat.class); ControlledJob controlledJob = new ControlledJob(job.getConfiguration()); jobControl.addJob(controlledJob);//ww w. ja v a2s. c o m Thread thread = new Thread(jobControl); thread.start(); while (!jobControl.allFinished()) { Thread.sleep(10000); } jobControl.stop(); FileSystem fs = FileSystem.get(job.getConfiguration()); FSDataInputStream in = fs.open(new Path(outputFilePath + "/part-r-00000")); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in)); String valueLine = bufferedReader.readLine(); String[] fields = valueLine.split("\t"); double value = Double.parseDouble(fields[1]); bufferedReader.close(); in.close(); return value; }
From source file:main.Driver.java
@Override public int run(String[] args) throws Exception { // TODO Auto-generated method stub if (args.length != 5) { System.out.println("usage: [input1] [output1] [input2] [output2] [finaloutput]"); System.exit(-1);// w ww .j a va2 s. c om } _configuration = this.getConf(); System.out.println(_configuration.get("Hello")); ControlledJob nameJob = setSortingJob(args[0], args[1], LicenseOutputFormat.NAMES); ControlledJob licenseJob = setSortingJob(args[2], args[3], LicenseOutputFormat.LICENSE); ControlledJob mrJob = setMRJob(args[1], args[3], args[4]); mrJob.addDependingJob(nameJob); mrJob.addDependingJob(licenseJob); JobControl jobControl = new JobControl("MyJob"); jobControl.addJob(nameJob); jobControl.addJob(licenseJob); jobControl.addJob(mrJob); //jobControl.run(); Thread thread = new Thread(jobControl); thread.start(); while (!jobControl.allFinished()) { System.out.println("Running"); Thread.sleep(5000); } System.out.println("<<<Done>>>"); return 0; }
From source file:ml.shifu.guagua.mapreduce.GuaguaMapReduceClient.java
License:Apache License
/** * Default constructor. Construct default JobControl instance. */ public GuaguaMapReduceClient() { this.jc = new JobControl(INIT_JOB_ID_PREFIX); }
From source file:tv.icntv.grade.film.dbcollect.TableConcurrencyJob.java
License:Apache License
@Override public int run(String[] strings) throws Exception { Configuration configuration = getConf(); String[] tables = configuration.get("hbase.cdn.tables").split(","); JobControl jobControl = new JobControl("data init"); for (String table : tables) { ///*w ww. j a v a 2 s. c om*/ String hbaseDbDirectory = String.format(configuration.get("hdfs.directory.from.hbase"), new Date(), table); HadoopUtils.deleteIfExist(hbaseDbDirectory); Job tableJob = new Job(configuration, "icntv grade init " + table); TableMapReduceUtil.initTableMapperJob(table, new Scan(), TableInitMapper.class, Text.class, Text.class, tableJob); MapReduceUtils.initReducerJob(new Path(hbaseDbDirectory), TableInitReducer.class, tableJob); // controlled job ControlledJob tableControlledJob = new ControlledJob(configuration); tableControlledJob.setJob(tableJob); String dbDirectory = String.format(configuration.get("hdfs.directory.base.db"), new Date(), table); HadoopUtils.deleteIfExist(dbDirectory); Configuration conf = getConf(); Job db = new Job(conf, "icntv db collect " + table); conf.setLong("mapred.min.split.size", 512 * 2014 * 1024L); MapReduceUtils.initMapperJob(DefaultHbaseMapper.class, Text.class, Text.class, this.getClass(), db, new Path(strings[1])); FileOutputFormat.setOutputPath(db, new Path(dbDirectory)); db.setNumReduceTasks(0); ControlledJob dbControlledJob = new ControlledJob(conf); dbControlledJob.setJob(db); dbControlledJob.addDependingJob(tableControlledJob); //controlledJob. jobControl.addJob(tableControlledJob); jobControl.addJob(dbControlledJob); } new Thread(jobControl).start(); while (!jobControl.allFinished()) { Thread.sleep(5000); } return 0; }
From source file:tv.icntv.grade.film.dbcollect.TableInitJob.java
License:Apache License
@Override public int run(String[] strings) throws Exception { Configuration configuration = getConf(); JobControl jobControl = new JobControl("init data"); for (String table : strings) { String dbPath = String.format(configuration.get("hdfs.directory.base.db"), new Date(), table); // String[] arrays = new String[]{table,//input table // String.format(configuration.get("hdfs.directory.from.hbase"), new Date(), table), // db // }; String hbasePath = String.format(configuration.get("hdfs.directory.from.hbase"), new Date(), table); //table job Job tableJob = new Job(configuration, "icntv grade init"); Scan scan = new Scan(); HadoopUtils.deleteIfExist(hbasePath); HadoopUtils.deleteIfExist(dbPath); TableMapReduceUtil.initTableMapperJob(table, scan, TableInitMapper.class, Text.class, Text.class, tableJob);/*from w w w . j a va 2 s . c o m*/ MapReduceUtils.initReducerJob(new Path(hbasePath), TableInitReducer.class, tableJob); ControlledJob firstControll = new ControlledJob(configuration); firstControll.setJob(tableJob); // tableJob.waitForCompletion(true); Job db = new Job(configuration, "icntv db collect"); configuration.setLong("mapred.min.split.size", 512 * 2014 * 1024L); MapReduceUtils.initMapperJob(DefaultHbaseMapper.class, Text.class, Text.class, this.getClass(), db, new Path(hbasePath)); FileOutputFormat.setOutputPath(db, new Path(dbPath)); db.setNumReduceTasks(0); ControlledJob secondaryController = new ControlledJob(configuration); secondaryController.setJob(db); secondaryController.addDependingJob(firstControll); jobControl.addJob(firstControll); jobControl.addJob(secondaryController); } new Thread(jobControl).start(); while (!jobControl.allFinished()) { Thread.sleep(5000); } logger.info("job controller successed job size=" + jobControl.getSuccessfulJobList().size()); // db.waitForCompletion(true); return 0; }
From source file:tv.icntv.grade.film.grade.GradeJob.java
License:Apache License
@Override public int run(String[] strings) throws Exception { Configuration configuration = this.getConf(); //num job//from www . ja v a 2 s . c o m Job numJob = new Job(configuration, "calculate film seed num job "); Path[] paths = getPaths(strings[0].split(",")); MapReduceUtils.initMapperJob(NumCountMapper.class, Text.class, LongWritable.class, this.getClass(), numJob, paths); // TableMapReduceUtil.initTableReducerJob(strings[1], NumCountReducer.class, numJob); ControlledJob controlledJob3 = new ControlledJob(configuration); controlledJob3.setJob(numJob); //time job Job timeJob = new Job(configuration, "calculate film time middle job"); MapReduceUtils.initMapperJob(TimeMaper.class, Text.class, Text.class, this.getClass(), timeJob, paths); timeJob.setCombinerClass(TimeCombiner.class); MapReduceUtils.initReducerJob(new Path(strings[2]), TimeReducer.class, timeJob); ControlledJob controlledJob4 = new ControlledJob(configuration); controlledJob4.setJob(timeJob); // //? Job correlate = new Job(configuration, "icntv correlate job"); MapReduceUtils.initMapperJob(UserHistoryMapper.class, Text.class, Text.class, this.getClass(), correlate, paths); MapReduceUtils.initReducerJob(new Path(strings[3]), UserHistoryReducer.class, correlate); ControlledJob correlateController = new ControlledJob(configuration); correlateController.setJob(correlate); // controlledJob3.getDependentJobs().add() JobControl jobControl = new JobControl("unit grade"); jobControl.addJob(controlledJob3); jobControl.addJob(controlledJob4); // jobControl.addJob(controlledJob5); jobControl.addJob(correlateController); new Thread(jobControl).start(); while (!jobControl.allFinished()) { Thread.sleep(5000); } // Counter counter =correlate.getCounters().findCounter(groupName,countName); // HadoopUtils.save(new Path(configuration.get("icntv.temp.file")),counter.getValue()); return 0; }
From source file:tv.icntv.log.stb.GenerateStbLogJob.java
License:Apache License
@Override public boolean run(Map<String, String> maps) throws Exception { //To change body of implemented methods use File | Settings | File Templates. Configuration configuration = getConf(); //user login// w ww .j a va 2s . c om Job userLogin = Job.getInstance(configuration, "user login job"); userLogin.setMapperClass(ParserLoginMapper.class); userLogin.setJarByClass(this.getClass()); userLogin.setOutputKeyClass(NullWritable.class); userLogin.setOutputValueClass(Text.class); Path userLoginOutput = new Path(maps.get(USER_LOGIN_JOB_OUTPUT)); FileInputFormat.addInputPath(userLogin, new Path(maps.get(USER_LOGIN_JOB_INPUT))); FileOutputFormat.setOutputPath(userLogin, userLoginOutput); userLogin.setNumReduceTasks(0); ControlledJob userControlledJob = new ControlledJob(configuration); userControlledJob.setJob(userLogin); // //player Job player = Job.getInstance(configuration, "player job"); player.setJarByClass(getClass()); player.setMapperClass(PlayerMapper.class); player.setOutputValueClass(Text.class); player.setOutputKeyClass(NullWritable.class); FileInputFormat.addInputPath(player, new Path(maps.get(PLAYER_JOB_INPUT))); FileOutputFormat.setOutputPath(player, new Path(maps.get(PLAYER_JOB_OUTPUT))); player.setNumReduceTasks(0); ControlledJob playControlledJob = new ControlledJob(configuration); playControlledJob.setJob(player); //contentView Job contentView = Job.getInstance(configuration, "content view job"); contentView.setJarByClass(getClass()); contentView.setMapperClass(ContentViewMapperBack.class); contentView.setOutputKeyClass(NullWritable.class); contentView.setOutputValueClass(Text.class); //System.out.println(MessageFormat.format(maps.get(OUTPUT_PREFIX),day)+MessageFormat.format(maps.get(CONTENT_VIEW_JOB_INPUT),configuration.getLong(FILTER_TIME,0L))); FileInputFormat.addInputPath(contentView, new Path(maps.get(CONTENT_VIEW_JOB_INPUT))); FileOutputFormat.setOutputPath(contentView, new Path(maps.get(CONTENT_VIEW_JOB_OUTPUT))); contentView.setNumReduceTasks(0); ControlledJob contentViewControlledJob = new ControlledJob(configuration); contentViewControlledJob.setJob(contentView); //reply Job replay = Job.getInstance(configuration, "reply job"); replay.setJarByClass(getClass()); replay.setMapperClass(ReplayMapper.class); replay.setOutputKeyClass(NullWritable.class); replay.setOutputValueClass(Text.class); FileInputFormat.addInputPath(replay, new Path(maps.get(LOOK_BACK_JOB_INPUT))); FileOutputFormat.setOutputPath(replay, new Path(maps.get(LOOK_BACK_JOB_OUTPUT))); replay.setNumReduceTasks(0); ControlledJob replayControlledJob = new ControlledJob(configuration); replayControlledJob.setJob(replay); //logEpg Job logEpg = Job.getInstance(configuration, "log epg job"); logEpg.setJarByClass(getClass()); logEpg.setMapperClass(EPGOperateMapper.class); logEpg.setOutputKeyClass(NullWritable.class); logEpg.setOutputValueClass(Text.class); FileInputFormat.addInputPath(logEpg, new Path(maps.get(LOG_EPG_JOB_INPUT))); FileOutputFormat.setOutputPath(logEpg, new Path(maps.get(LOG_EPG_JOB_OUTPUT))); logEpg.setNumReduceTasks(0); ControlledJob logEpgControlledJob = new ControlledJob(configuration); logEpgControlledJob.setJob(logEpg); // //cdn // Job cdn=Job.getInstance(configuration,"cdn job"); // cdn.setJarByClass(getClass()); // cdn.setMapperClass(CdnModuleMapper.class); // cdn.setOutputKeyClass(NullWritable.class); // cdn.setOutputValueClass(Text.class); // FileInputFormat.addInputPath(cdn, new Path(maps.get(CDN_JOB_INPUT))); // FileOutputFormat.setOutputPath(cdn, new Path(maps.get(CDN_JOB_OUTPUT))); // cdn.setNumReduceTasks(0); // ControlledJob cdnControlledJob=new ControlledJob(configuration); // cdnControlledJob.setJob(cdn); //cdn stb Job cdn = Job.getInstance(configuration, "cdn stb job"); cdn.setJarByClass(this.getClass()); cdn.setMapperClass(CdnStbMapper.class); cdn.setOutputValueClass(Text.class); cdn.setOutputKeyClass(NullWritable.class); FileInputFormat.addInputPath(cdn, new Path(maps.get(CDN_JOB_INPUT))); FileOutputFormat.setOutputPath(cdn, new Path(maps.get(CDN_JOB_OUTPUT))); cdn.setNumReduceTasks(0); ControlledJob cdnControlledJob = new ControlledJob(configuration); cdnControlledJob.setJob(cdn); //cdn adapter Job cdnAdapterJob = Job.getInstance(configuration, "cdn adapter job "); cdnAdapterJob.setJarByClass(getClass()); cdnAdapterJob.setMapperClass(CdnAdapterMapper.class); cdnAdapterJob.setOutputKeyClass(NullWritable.class); cdnAdapterJob.setOutputValueClass(Text.class); FileInputFormat.addInputPath(cdnAdapterJob, new Path(maps.get(CDN_ADAPTER_JOB_INPUT))); FileOutputFormat.setOutputPath(cdnAdapterJob, new Path(maps.get(CDN_ADAPTER_JOB_OUTPUT))); cdnAdapterJob.setNumReduceTasks(0); ControlledJob cdnAdapterControlleredJob = new ControlledJob(configuration); cdnAdapterControlleredJob.setJob(cdnAdapterJob); JobControl jobControl = new JobControl("stb log parser .eg: userLogin,devicePlayer,contentView,logEpg,cdn"); jobControl.addJob(userControlledJob); jobControl.addJob(playControlledJob); jobControl.addJob(contentViewControlledJob); jobControl.addJob(replayControlledJob); jobControl.addJob(logEpgControlledJob); jobControl.addJob(cdnControlledJob); jobControl.addJob(cdnAdapterControlleredJob); new Thread(jobControl).start(); while (!jobControl.allFinished()) { Thread.sleep(5000); } if (jobControl.getFailedJobList().size() > 0) { return false; } return true; }