Example usage for org.apache.hadoop.mapred JobStatus RUNNING

List of usage examples for org.apache.hadoop.mapred JobStatus RUNNING

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.mapred.*
  5. JobStatus
  6. RUNNING

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred JobStatus RUNNING.

Prototype

int RUNNING

To view the source code for org.apache.hadoop.mapred JobStatus RUNNING.

Click Source Link

Usage

From source file:com.chinamobile.bcbsp.bspstaff.StaffInProgress.java

License:Apache License

/**
 * Judge if the staff should close.//from   w  ww .j a va2  s.c om
 * @param sid the staff id
 * @return true:the staff should close;false:the staff should not close
 */
public boolean shouldCloseForClosedJob(StaffAttemptID sid) {
    StaffStatus ss = staffStatuses.get(sid);
    if ((ss != null) && (!staffReportedClosed.contains(sid))
            && (job.getStatus().getRunState() != JobStatus.RUNNING)) {
        staffReportedClosed.add(sid);
        return true;
    } else {
        return false;
    }
}

From source file:com.jackbe.mapreduce.LocalJobManager.java

License:Open Source License

@Override
public void clearCompletedJobs() {
    if (log.isTraceEnabled())
        log.trace("called.");

    for (String key : statusMap.keySet()) {
        RunningJob job = statusMap.get(key);
        try {/*  ww  w. j  a  va  2s.  co m*/
            if (job.getJobState() != JobStatus.RUNNING)
                statusMap.remove(key);
        } catch (IOException e) {
            log.error("Exception clearing completed jobs: " + e, e);
        }
    }
}

From source file:com.liferay.hadoop.action.HadoopJob.java

License:Open Source License

public String doExecute(HttpServletRequest request, HttpServletResponse response) throws Exception {

    response.setContentType(ContentTypes.TEXT_PLAIN_UTF8);

    PrintWriter writer = response.getWriter();

    FileSystem fileSystem = HadoopManager.getFileSystem();

    JobClient jobClient = HadoopManager.getJobClient();

    writer.println("-- Job Status --");

    Path inputPath = new Path("/index/*/*");
    Path outputPath = new Path("/wordcount/results");

    try {//from w  w  w  .  j a va2 s  .c om
        if (_runningJob == null) {
            writer.println("Creating job");

            if (fileSystem.exists(_jobPath)) {
                fileSystem.delete(_jobPath, false);
            }

            if (!fileSystem.exists(_jobPath)) {
                writer.println("Deploying the job code to cluster");

                FSDataOutputStream outputStream = null;

                try {
                    outputStream = fileSystem.create(_jobPath);

                    ServletContext servletContext = HadoopManager.getServletContext();

                    InputStream inputStream = servletContext.getResourceAsStream("/WEB-INF/lib/hadoop-job.jar");

                    StreamUtil.transfer(inputStream, outputStream, false);
                } finally {
                    StreamUtil.cleanUp(outputStream);
                }

                writer.println("Job code deployed to cluster");
            }

            if (fileSystem.exists(outputPath)) {
                writer.println("A previous job output was found, backing it up");

                fileSystem.rename(outputPath,
                        outputPath.getParent().suffix("/.results-" + System.currentTimeMillis()));
            }

            _jobConf = HadoopManager.createNewJobConf();

            _jobConf.setJobName("Word Count");

            writer.println("Job '" + _jobConf.getJobName() + "' is being configured");

            _jobConf.setJarByClass(Map.class);
            _jobConf.setOutputKeyClass(Text.class);
            _jobConf.setOutputValueClass(IntWritable.class);
            _jobConf.setMapperClass(Map.class);
            _jobConf.setCombinerClass(Reduce.class);
            _jobConf.setReducerClass(Reduce.class);
            _jobConf.setInputFormat(TextInputFormat.class);
            _jobConf.setOutputFormat(TextOutputFormat.class);

            writer.println("Job code deployed to distributed cache's classpath");

            DistributedCache.addArchiveToClassPath(_jobPath, _jobConf, fileSystem);

            FileInputFormat.setInputPaths(_jobConf, inputPath);
            FileOutputFormat.setOutputPath(_jobConf, outputPath);

            writer.println("Submitting job the first time");

            _runningJob = jobClient.submitJob(_jobConf);

            writer.println("Job submitted");
        }

        int jobState = _runningJob.getJobState();

        writer.println(
                "Job status: " + jobState + " (RUNNING = 1, SUCCEEDED = 2, FAILED = 3, PREP = 4, KILLED = 5)");

        if ((jobState != JobStatus.RUNNING) && (jobState != JobStatus.PREP)) {

            writer.println("Re-issuing the job");

            if (fileSystem.exists(outputPath)) {
                writer.println("A previous job output was found, backing it up");

                fileSystem.rename(outputPath,
                        outputPath.getParent().suffix("/.results-" + System.currentTimeMillis()));
            }

            writer.println("Submitting job the first time");

            _runningJob = jobClient.submitJob(_jobConf);

            writer.println("Job submitted");
        }
    } catch (Exception ioe) {
        writer.println("Job error: ");

        ioe.printStackTrace(writer);
    }

    writer.flush();
    writer.close();

    return null;
}

From source file:com.liferay.hadoop.util.HadoopManager.java

License:Open Source License

public static void runJob(StoreEvent storeEvent) throws IOException {
    FileSystem fileSystem = getFileSystem();

    if (_servletContext == null) {
        return;//from  www .j  a  v  a  2s . c  om
    }

    JobClient jobClient = getJobClient();

    Path inputPath = new Path("/index".concat(storeEvent.getRootPath().toString()).concat("/*"));
    Path outputPath = new Path("/wordcount".concat(storeEvent.getRootPath().toString()).concat("/results"));

    try {
        if (_runningJob == null) {
            if (!fileSystem.exists(_jobPath)) {
                FSDataOutputStream outputStream = null;

                try {
                    outputStream = fileSystem.create(_jobPath);

                    InputStream inputStream = _servletContext
                            .getResourceAsStream("/WEB-INF/lib/hadoop-job.jar");

                    StreamUtil.transfer(inputStream, outputStream, false);
                } finally {
                    StreamUtil.cleanUp(outputStream);
                }
            }

            if (fileSystem.exists(outputPath)) {
                fileSystem.rename(outputPath,
                        outputPath.getParent().suffix("/.results-" + System.currentTimeMillis()));
            }

            _jobConf = new JobConf(_sharedJobConf);

            _jobConf.setJobName("Word Count");
            _jobConf.setJarByClass(Map.class);
            _jobConf.setOutputKeyClass(Text.class);
            _jobConf.setOutputValueClass(IntWritable.class);
            _jobConf.setMapperClass(Map.class);
            _jobConf.setCombinerClass(Reduce.class);
            _jobConf.setReducerClass(Reduce.class);
            _jobConf.setInputFormat(TextInputFormat.class);
            _jobConf.setOutputFormat(TextOutputFormat.class);

            DistributedCache.addArchiveToClassPath(_jobPath, _jobConf, fileSystem);

            FileInputFormat.setInputPaths(_jobConf, inputPath);
            FileOutputFormat.setOutputPath(_jobConf, outputPath);

            _runningJob = jobClient.submitJob(_jobConf);
        }

        int jobState = _runningJob.getJobState();

        if ((jobState != JobStatus.RUNNING) && (jobState != JobStatus.PREP)) {

            System.out.println("Re-issuing the word count job.");

            if (fileSystem.exists(outputPath)) {
                fileSystem.rename(outputPath,
                        outputPath.getParent().suffix("/.results-" + System.currentTimeMillis()));
            }

            _runningJob = jobClient.submitJob(_jobConf);
        }
    } catch (Exception ioe) {
        ioe.printStackTrace();
    }
}

From source file:com.ngdata.hbaseindexer.master.BatchStateUpdater.java

License:Apache License

@Override
public void run() {
    IndexerDefinition indexerDefinition = null;
    try {//from   w w  w .j a va 2s .c o m
        indexerDefinition = indexerModel.getIndexer(indexerName);
    } catch (IndexerNotFoundException e) {
        log.info("Could not find index " + indexerName + " while checking batch rebuild status.", e);
    }
    if (indexerDefinition != null) {
        log.debug("Checking batch state for " + indexerDefinition.getName());
        BatchBuildInfo batchBuildInfo = indexerDefinition.getActiveBatchBuildInfo();
        if (batchBuildInfo != null) {
            Set<String> jobs = batchBuildInfo.getMapReduceJobTrackingUrls().keySet();

            boolean batchDone = true;
            boolean overAllSuccess = true;
            for (String jobId : jobs) {
                RunningJob job;
                try {
                    job = jobClient.getJob(JobID.forName(jobId));
                } catch (IOException e) {
                    log.error("Could not get job " + jobId + " for index " + indexerDefinition.getName()
                            + " while checking active build info.", e);
                    batchDone = false;
                    break;
                }
                if (job != null) {
                    int jobState;
                    try {
                        jobState = job.getJobState();
                    } catch (IOException e) {
                        log.error(
                                "Could not get jobstate for job " + jobId + " for index "
                                        + indexerDefinition.getName() + " while checking active build info.",
                                e);
                        batchDone = false;
                        break;
                    }
                    batchDone = batchDone && jobState != JobStatus.RUNNING;
                    overAllSuccess = overAllSuccess && jobState == JobStatus.SUCCEEDED;
                } else {
                    log.warn("Could not find job " + jobId + " while checking active batch builds for indexer "
                            + indexerDefinition.getName());
                }
            }
            if (batchDone) {
                markBatchBuildCompleted(indexerDefinition.getName(), overAllSuccess);
            } else {
                executor.schedule(this, pollInterval, TimeUnit.MILLISECONDS);
            }
        }
    }
}

From source file:com.ngdata.hbaseindexer.master.BatchStateUpdaterTest.java

License:Apache License

@Test
public void testRun_Running() throws Exception {
    String jobId = "job_201407251005_0815";
    createDefinition("mytest", jobId);
    RunningJob job = createJob(jobId, JobStatus.RUNNING);

    when(job.getJobState()).thenReturn(JobStatus.RUNNING);

    Assert.assertEquals(0, executorService.getQueue().size());
    checkAllIndexes();//from  w  w  w  .  j a va  2s . co  m

    Assert.assertEquals(1, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(1)).getIndexer(anyString());
    verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class));
    Thread.sleep(60);
    Assert.assertEquals(1, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(2)).getIndexer(anyString());
    verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class));

    when(job.getJobState()).thenReturn(JobStatus.SUCCEEDED);
    Thread.sleep(60);
    Assert.assertEquals(0, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(3)).getIndexer(anyString());
    verify(model, VerificationModeFactory.times(1)).updateIndexerInternal(any(IndexerDefinition.class));
}

From source file:com.ngdata.hbaseindexer.master.BatchStateUpdaterTest.java

License:Apache License

@Test
public void testRun_MultiIndex() throws Exception {
    String jobId1 = "job_201407251005_0001";
    String jobId2 = "job_201407251005_0002";
    createDefinition("mytest1", jobId1);
    RunningJob job1 = createJob(jobId1, JobStatus.RUNNING);
    createDefinition("mytest2", jobId2);
    RunningJob job2 = createJob(jobId2, JobStatus.RUNNING);

    Assert.assertEquals(0, executorService.getQueue().size());
    checkAllIndexes();/*w w  w  .j  a  va 2 s  .c om*/

    Assert.assertEquals(2, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class));
    Thread.sleep(60);
    Assert.assertEquals(2, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class));

    when(job1.getJobState()).thenReturn(JobStatus.SUCCEEDED);
    Thread.sleep(60);
    Assert.assertEquals(1, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(1)).updateIndexerInternal(any(IndexerDefinition.class));

    when(job2.getJobState()).thenReturn(JobStatus.SUCCEEDED);
    Thread.sleep(60);
    Assert.assertEquals(0, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(2)).updateIndexerInternal(any(IndexerDefinition.class));

    Thread.sleep(60);
    Assert.assertEquals(0, executorService.getQueue().size());
    verify(model, VerificationModeFactory.times(2)).updateIndexerInternal(any(IndexerDefinition.class));
}

From source file:org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine.java

License:Apache License

/**
 * Convert map-reduce specific job status constants to Sqoop job status
 * constants.//from ww  w .j a va  2 s  .  c  om
 *
 * @param status Map-reduce job constant
 * @return Equivalent submission status
 */
protected SubmissionStatus convertMapreduceState(int status) {
    if (status == JobStatus.PREP) {
        return SubmissionStatus.BOOTING;
    } else if (status == JobStatus.RUNNING) {
        return SubmissionStatus.RUNNING;
    } else if (status == JobStatus.FAILED) {
        return SubmissionStatus.FAILED;
    } else if (status == JobStatus.KILLED) {
        return SubmissionStatus.FAILED;
    } else if (status == JobStatus.SUCCEEDED) {
        return SubmissionStatus.SUCCEEDED;
    }

    throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0004, "Unknown status " + status);
}

From source file:org.estado.core.JobStatusChecker.java

License:Apache License

public void checkStatus() {
    List<org.estado.spi.JobStatus> jobStatusList = new ArrayList<org.estado.spi.JobStatus>();

    try {/*from  ww  w  .j a  v a2 s  .c o m*/
        Configuration conf = new Configuration();
        JobClient client = new JobClient(new JobConf(conf));
        JobStatus[] jobStatuses = client.getAllJobs();
        showFilter();

        int jobCount = 0;
        for (JobStatus jobStatus : jobStatuses) {
            Long lastTaskEndTime = 0L;
            TaskReport[] mapReports = client.getMapTaskReports(jobStatus.getJobID());
            for (TaskReport r : mapReports) {
                if (lastTaskEndTime < r.getFinishTime()) {
                    lastTaskEndTime = r.getFinishTime();
                }
            }
            TaskReport[] reduceReports = client.getReduceTaskReports(jobStatus.getJobID());
            for (TaskReport r : reduceReports) {
                if (lastTaskEndTime < r.getFinishTime()) {
                    lastTaskEndTime = r.getFinishTime();
                }
            }
            client.getSetupTaskReports(jobStatus.getJobID());
            client.getCleanupTaskReports(jobStatus.getJobID());

            String jobId = jobStatus.getJobID().toString();
            String jobName = client.getJob(jobStatus.getJobID()).getJobName();
            Long startTime = jobStatus.getStartTime();
            String user = jobStatus.getUsername();
            int mapProgress = (int) (jobStatus.mapProgress() * 100);
            int reduceProgress = (int) (jobStatus.reduceProgress() * 100);
            org.estado.spi.JobStatus jobStat = null;
            ++jobCount;

            int runState = jobStatus.getRunState();
            switch (runState) {
            case JobStatus.SUCCEEDED:
                if (filter.contains("s")) {
                    Long duration = lastTaskEndTime - jobStatus.getStartTime();
                    jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime,
                            lastTaskEndTime, duration, mapProgress, reduceProgress, "completed");
                    ++sCount;
                }
                break;

            case JobStatus.RUNNING:
                if (filter.contains("r")) {
                    long duration = System.currentTimeMillis() - jobStatus.getStartTime();
                    jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime,
                            lastTaskEndTime, duration, mapProgress, reduceProgress, "running");
                    ++rCount;
                }
                break;

            case JobStatus.FAILED:
                if (filter.contains("f")) {
                    long duration = lastTaskEndTime - jobStatus.getStartTime();
                    jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime,
                            lastTaskEndTime, duration, mapProgress, reduceProgress, "failed");
                    RunningJob job = client.getJob(jobStatus.getJobID());
                    jobStat.setJobTasks(getTaskDetails(job));
                    ++fCount;
                }
                break;

            case JobStatus.PREP:
                if (filter.contains("p")) {
                    jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, null, null,
                            null, 0, 0, "preparing");
                    ++pCount;
                }
                break;

            case JobStatus.KILLED:
                if (filter.contains("k")) {
                    long duration = lastTaskEndTime - jobStatus.getStartTime();

                    jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime,
                            lastTaskEndTime, duration, mapProgress, reduceProgress, "killed");

                    RunningJob job = client.getJob(jobStatus.getJobID());
                    jobStat.setJobTasks(getTaskDetails(job));
                    ++kCount;
                }
                break;
            }

            jobStatusList.add(jobStat);
        }

        //get counters
        for (org.estado.spi.JobStatus jobStat : jobStatusList) {
            if (!jobStat.getStatus().equals("preparing")) {
                List<JobCounterGroup> counterGroups = getJobCounters(jobStat.getJobId());
                jobStat.setCounterGroups(counterGroups);

                //additional data from counters
                setJobInfo(jobStat);
            }
        }

        //publish to all consumers
        for (JobStatusConsumer consumer : consumers) {
            consumer.handle(jobStatusList);
        }

        showJobCounts();
    } catch (Exception ex) {
        System.out.println("Jobs status checker failed" + ex.getMessage());
    }

}

From source file:org.godhuli.rhipe.FileUtils.java

License:Apache License

public REXP getstatus(String jd, boolean geterrors) throws Exception {
    org.apache.hadoop.mapred.JobID jj = org.apache.hadoop.mapred.JobID.forName(jd);
    if (jj == null)
        throw new IOException("Jobtracker could not find jobID: " + jd);
    org.apache.hadoop.mapred.RunningJob rj = jclient.getJob(jj);
    if (rj == null)
        throw new IOException(
                "No such job: " + jd + " available, wrong job? or try the History Viewer (see the Web UI) ");
    String jobfile = rj.getJobFile();
    String jobname = rj.getJobName();
    // cfg.addResource(new Path(jobfile));
    org.apache.hadoop.mapred.Counters cc = rj.getCounters();
    long startsec = getStart(jclient, jj);
    double dura = ((double) System.currentTimeMillis() - startsec) / 1000;
    REXP ro = FileUtils.buildlistFromOldCounter(cc, dura);
    int jobs = rj.getJobState();
    String jobss = null;//  w ww.  j  ava 2 s  . c o m
    if (jobs == JobStatus.FAILED)
        jobss = "FAILED";
    else if (jobs == JobStatus.KILLED)
        jobss = "KILLED";
    else if (jobs == JobStatus.PREP)
        jobss = "PREP";
    else if (jobs == JobStatus.RUNNING)
        jobss = "RUNNING";
    else if (jobs == JobStatus.SUCCEEDED)
        jobss = "SUCCEEDED";
    float mapprog = rj.mapProgress(), reduprog = rj.reduceProgress();

    org.apache.hadoop.mapred.TaskReport[] maptr = jclient.getMapTaskReports(jj);
    org.apache.hadoop.mapred.TaskReport[] redtr = jclient.getReduceTaskReports(jj);

    int totalmaps = maptr.length, totalreds = redtr.length;
    int mappending = 0, redpending = 0, maprunning = 0, redrunning = 0, redfailed = 0, redkilled = 0,
            mapkilled = 0, mapfailed = 0, mapcomp = 0, redcomp = 0;
    for (int i = 0; i < maptr.length; i++) {
        TIPStatus t = maptr[i].getCurrentStatus();
        switch (t) {
        case COMPLETE:
            mapcomp++;
            break;
        case FAILED:
            mapfailed++;
            break;
        case PENDING:
            mappending++;
            break;
        case RUNNING:
            maprunning++;
            break;
        case KILLED:
            mapkilled++;
            break;
        }
    }
    for (int i = 0; i < redtr.length; i++) {
        TIPStatus t = redtr[i].getCurrentStatus();
        switch (t) {
        case COMPLETE:
            redcomp++;
            break;
        case FAILED:
            redfailed++;
            break;
        case PENDING:
            redpending++;
            break;
        case RUNNING:
            redrunning++;
            break;
        case KILLED:
            redkilled++;
            break;
        }
    }
    int reduceafails = 0, reduceakilled = 0, mapafails = 0, mapakilled = 0;
    int startfrom = 0;

    REXP.Builder errcontainer = REXP.newBuilder();
    errcontainer.setRclass(REXP.RClass.STRING);
    while (true) {
        org.apache.hadoop.mapred.TaskCompletionEvent[] events = rj.getTaskCompletionEvents(startfrom);
        for (int i = 0; i < events.length; i++) {
            org.apache.hadoop.mapred.TaskCompletionEvent e = events[i];
            int f = 0, k = 0;
            switch (e.getTaskStatus()) {
            case KILLED:
                if (e.isMapTask()) {
                    mapakilled++;
                } else {
                    reduceakilled++;
                }
                break;
            case TIPFAILED:
            case FAILED:
                if (e.isMapTask()) {
                    mapafails++;
                } else {
                    reduceafails++;
                }
                if (geterrors) {
                    REXPProtos.STRING.Builder content = REXPProtos.STRING.newBuilder();
                    String[] s = rj.getTaskDiagnostics(e.getTaskAttemptId());
                    if (s != null && s.length > 0) {
                        content.setStrval(s[0]);
                        errcontainer.addStringValue(content.build());
                    }
                }
                break;
            }
        }
        startfrom += events.length;
        if (events.length == 0)
            break;
    }

    REXP.Builder thevals = REXP.newBuilder();
    thevals.setRclass(REXP.RClass.LIST);
    thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobss }));
    thevals.addRexpValue(RObjects.buildDoubleVector(new double[] { dura }));
    thevals.addRexpValue(RObjects.buildDoubleVector(new double[] { (double) mapprog, (double) reduprog }));
    thevals.addRexpValue(RObjects.buildIntVector(
            new int[] { totalmaps, mappending, maprunning, mapcomp, mapkilled, mapafails, mapakilled }));
    thevals.addRexpValue(RObjects.buildIntVector(
            new int[] { totalreds, redpending, redrunning, redcomp, redkilled, reduceafails, reduceakilled }));
    thevals.addRexpValue(ro);
    thevals.addRexpValue(errcontainer);
    thevals.addRexpValue(RObjects.makeStringVector(rj.getTrackingURL()));
    thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobname }));
    thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobfile }));
    return (thevals.build());
}