Example usage for org.apache.hadoop.mapreduce.lib.jobcontrol ControlledJob ControlledJob

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.lib.jobcontrol ControlledJob ControlledJob.

Prototype

public ControlledJob(Job job, List<ControlledJob> dependingJobs) throws IOException 

Source Link

Document

Construct a job.

Usage

From source file:ml.shifu.guagua.mapreduce.GuaguaMapReduceClient.java

License:Apache License

/**
 * Add new job to JobControl instance.//from w  w  w.  ja v a  2  s  . com
 */
public synchronized void addJob(String[] args) throws IOException {
    Job job = createJob(args);
    this.jc.addJob(new ControlledJob(job, null));
    if (this.jobIndexMap.containsKey(job.getJobName())) {
        throw new IllegalStateException(
                "Job name should be unique. please check name with: " + job.getJobName());
    }
    this.jobIndexMap.put(job.getJobName(), this.jobIndex);
    this.jobIndexParams.put(this.jobIndex, args);
    this.jobRunningTimes.put(this.jobIndex, 1);
    this.jobIndex += 1;
}

---

From source file:ml.shifu.guagua.mapreduce.GuaguaMapReduceClient.java

License:Apache License

/**
 * Run all jobs added to JobControl./*  w w  w . ja va 2  s. c om*/
 */
public void run() throws IOException {
    // Initially, all jobs are in wait state.
    List<ControlledJob> jobsWithoutIds = this.jc.getWaitingJobList();
    int totalNeededMRJobs = jobsWithoutIds.size();
    LOG.info("{} map-reduce job(s) waiting for submission.", jobsWithoutIds.size());
    Thread jcThread = new Thread(this.jc, "Guagua-MapReduce-JobControl");
    jcThread.start();

    JobClient jobClient = new JobClient(new JobConf(new Configuration()));
    double lastProg = -1;

    Set<String> sucessfulJobs = new HashSet<String>();

    while (!this.jc.allFinished()) {
        try {
            jcThread.join(1000);
        } catch (InterruptedException ignore) {
            Thread.currentThread().interrupt();
        }
        List<ControlledJob> jobsAssignedIdInThisRun = new ArrayList<ControlledJob>(totalNeededMRJobs);

        for (ControlledJob job : jobsWithoutIds) {
            if (job.getJob().getJobID() != null) {
                jobsAssignedIdInThisRun.add(job);
                LOG.info("Job {} is started.", job.getJob().getJobID().toString());
            } else {
                // This job is not assigned an id yet.
            }
        }
        jobsWithoutIds.removeAll(jobsAssignedIdInThisRun);

        List<ControlledJob> successfulJobs = jc.getSuccessfulJobList();
        for (ControlledJob controlledJob : successfulJobs) {
            String jobId = controlledJob.getJob().getJobID().toString();
            if (!sucessfulJobs.contains(jobId)) {
                LOG.info("Job {} is successful.", jobId);
                sucessfulJobs.add(jobId);
            }
        }

        List<ControlledJob> failedJobs = jc.getFailedJobList();
        for (ControlledJob controlledJob : failedJobs) {
            String failedJobId = controlledJob.getJob().getJobID().toString();
            if (!this.failedCheckingJobs.contains(failedJobId)) {
                this.failedCheckingJobs.add(failedJobId);
                String jobName = controlledJob.getJob().getJobName();
                Integer jobIndex = this.jobIndexMap.get(jobName);
                Integer runTimes = this.jobRunningTimes.get(jobIndex);
                if (runTimes <= 1) {
                    LOG.warn("Job {} is failed, will be submitted again.", jobName);
                    Job newJob = createJob(this.jobIndexParams.get(jobIndex));
                    this.jc.addJob(new ControlledJob(newJob, null));
                    this.jobRunningTimes.put(jobIndex, runTimes + 1);
                    this.jobIndexMap.put(newJob.getJobName(), jobIndex);
                    jobsWithoutIds = this.jc.getWaitingJobList();
                } else {
                    LOG.warn("Job {} is failed twice, will not be submitted again.", jobName);
                }
            }
        }
        double prog = calculateProgress(jc, jobClient) / totalNeededMRJobs;
        notifyProgress(prog, lastProg);
        lastProg = prog;

        try {
            Thread.sleep(2 * 1000);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }

    List<ControlledJob> successfulJobs = jc.getSuccessfulJobList();

    LOG.info("Sucessful jobs:");
    for (ControlledJob controlledJob : successfulJobs) {
        LOG.info("Job: {} ", controlledJob);
    }
    if (totalNeededMRJobs == successfulJobs.size()) {
        LOG.info("Guagua jobs: 100% complete");
        // add failed jobs to debug since all jobs are finished.
        List<ControlledJob> failedJobs = jc.getFailedJobList();
        if (failedJobs != null && failedJobs.size() > 0) {
            LOG.info("Failed jobs:");
            for (ControlledJob controlledJob : failedJobs) {
                LOG.debug("Job: {} ", controlledJob);
            }
        }
    } else {
        List<ControlledJob> failedJobs = jc.getFailedJobList();
        if (failedJobs != null && failedJobs.size() > 0) {
            LOG.info("Failed jobs:");
            for (ControlledJob controlledJob : failedJobs) {
                LOG.warn("Job: {} ", controlledJob);
            }
        }
    }
    this.jc.stop();
}

---