Example usage for org.apache.hadoop.mapreduce.lib.jobcontrol ControlledJob ControlledJob

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.jobcontrol ControlledJob ControlledJob.

Prototype

public ControlledJob(Configuration conf) throws IOException

Source Link

Document

Construct a job.

Usage

From source file:tv.icntv.log.stb.GenerateStbLogJob.java

License:Apache License

@Override
public boolean run(Map<String, String> maps) throws Exception {
    //To change body of implemented methods use File | Settings | File Templates.

    Configuration configuration = getConf();
    //user login//from  w  w  w  .j  a  v  a2 s. com
    Job userLogin = Job.getInstance(configuration, "user login job");
    userLogin.setMapperClass(ParserLoginMapper.class);
    userLogin.setJarByClass(this.getClass());
    userLogin.setOutputKeyClass(NullWritable.class);
    userLogin.setOutputValueClass(Text.class);
    Path userLoginOutput = new Path(maps.get(USER_LOGIN_JOB_OUTPUT));
    FileInputFormat.addInputPath(userLogin, new Path(maps.get(USER_LOGIN_JOB_INPUT)));
    FileOutputFormat.setOutputPath(userLogin, userLoginOutput);
    userLogin.setNumReduceTasks(0);
    ControlledJob userControlledJob = new ControlledJob(configuration);
    userControlledJob.setJob(userLogin);
    //
    //player
    Job player = Job.getInstance(configuration, "player job");
    player.setJarByClass(getClass());
    player.setMapperClass(PlayerMapper.class);
    player.setOutputValueClass(Text.class);
    player.setOutputKeyClass(NullWritable.class);
    FileInputFormat.addInputPath(player, new Path(maps.get(PLAYER_JOB_INPUT)));
    FileOutputFormat.setOutputPath(player, new Path(maps.get(PLAYER_JOB_OUTPUT)));
    player.setNumReduceTasks(0);
    ControlledJob playControlledJob = new ControlledJob(configuration);
    playControlledJob.setJob(player);

    //contentView
    Job contentView = Job.getInstance(configuration, "content view job");
    contentView.setJarByClass(getClass());
    contentView.setMapperClass(ContentViewMapperBack.class);
    contentView.setOutputKeyClass(NullWritable.class);
    contentView.setOutputValueClass(Text.class);
    //System.out.println(MessageFormat.format(maps.get(OUTPUT_PREFIX),day)+MessageFormat.format(maps.get(CONTENT_VIEW_JOB_INPUT),configuration.getLong(FILTER_TIME,0L)));
    FileInputFormat.addInputPath(contentView, new Path(maps.get(CONTENT_VIEW_JOB_INPUT)));
    FileOutputFormat.setOutputPath(contentView, new Path(maps.get(CONTENT_VIEW_JOB_OUTPUT)));
    contentView.setNumReduceTasks(0);
    ControlledJob contentViewControlledJob = new ControlledJob(configuration);
    contentViewControlledJob.setJob(contentView);

    //reply
    Job replay = Job.getInstance(configuration, "reply job");
    replay.setJarByClass(getClass());
    replay.setMapperClass(ReplayMapper.class);
    replay.setOutputKeyClass(NullWritable.class);
    replay.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(replay, new Path(maps.get(LOOK_BACK_JOB_INPUT)));
    FileOutputFormat.setOutputPath(replay, new Path(maps.get(LOOK_BACK_JOB_OUTPUT)));
    replay.setNumReduceTasks(0);
    ControlledJob replayControlledJob = new ControlledJob(configuration);
    replayControlledJob.setJob(replay);

    //logEpg
    Job logEpg = Job.getInstance(configuration, "log epg job");
    logEpg.setJarByClass(getClass());
    logEpg.setMapperClass(EPGOperateMapper.class);
    logEpg.setOutputKeyClass(NullWritable.class);
    logEpg.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(logEpg, new Path(maps.get(LOG_EPG_JOB_INPUT)));
    FileOutputFormat.setOutputPath(logEpg, new Path(maps.get(LOG_EPG_JOB_OUTPUT)));
    logEpg.setNumReduceTasks(0);
    ControlledJob logEpgControlledJob = new ControlledJob(configuration);
    logEpgControlledJob.setJob(logEpg);

    //        //cdn
    //        Job cdn=Job.getInstance(configuration,"cdn job");
    //        cdn.setJarByClass(getClass());
    //        cdn.setMapperClass(CdnModuleMapper.class);
    //        cdn.setOutputKeyClass(NullWritable.class);
    //        cdn.setOutputValueClass(Text.class);
    //        FileInputFormat.addInputPath(cdn, new Path(maps.get(CDN_JOB_INPUT)));
    //        FileOutputFormat.setOutputPath(cdn, new Path(maps.get(CDN_JOB_OUTPUT)));
    //        cdn.setNumReduceTasks(0);
    //        ControlledJob cdnControlledJob=new ControlledJob(configuration);
    //        cdnControlledJob.setJob(cdn);
    //cdn stb
    Job cdn = Job.getInstance(configuration, "cdn stb job");
    cdn.setJarByClass(this.getClass());
    cdn.setMapperClass(CdnStbMapper.class);
    cdn.setOutputValueClass(Text.class);
    cdn.setOutputKeyClass(NullWritable.class);
    FileInputFormat.addInputPath(cdn, new Path(maps.get(CDN_JOB_INPUT)));
    FileOutputFormat.setOutputPath(cdn, new Path(maps.get(CDN_JOB_OUTPUT)));
    cdn.setNumReduceTasks(0);
    ControlledJob cdnControlledJob = new ControlledJob(configuration);
    cdnControlledJob.setJob(cdn);
    //cdn adapter

    Job cdnAdapterJob = Job.getInstance(configuration, "cdn adapter job ");
    cdnAdapterJob.setJarByClass(getClass());
    cdnAdapterJob.setMapperClass(CdnAdapterMapper.class);
    cdnAdapterJob.setOutputKeyClass(NullWritable.class);
    cdnAdapterJob.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(cdnAdapterJob, new Path(maps.get(CDN_ADAPTER_JOB_INPUT)));
    FileOutputFormat.setOutputPath(cdnAdapterJob, new Path(maps.get(CDN_ADAPTER_JOB_OUTPUT)));
    cdnAdapterJob.setNumReduceTasks(0);
    ControlledJob cdnAdapterControlleredJob = new ControlledJob(configuration);
    cdnAdapterControlleredJob.setJob(cdnAdapterJob);

    JobControl jobControl = new JobControl("stb log parser .eg: userLogin,devicePlayer,contentView,logEpg,cdn");
    jobControl.addJob(userControlledJob);
    jobControl.addJob(playControlledJob);
    jobControl.addJob(contentViewControlledJob);
    jobControl.addJob(replayControlledJob);
    jobControl.addJob(logEpgControlledJob);
    jobControl.addJob(cdnControlledJob);
    jobControl.addJob(cdnAdapterControlleredJob);
    new Thread(jobControl).start();
    while (!jobControl.allFinished()) {
        Thread.sleep(5000);
    }
    if (jobControl.getFailedJobList().size() > 0) {
        return false;
    }
    return true;
}