Example usage for org.apache.spark JobExecutionStatus RUNNING

Introduction

In this page you can find the example usage for org.apache.spark JobExecutionStatus RUNNING.

Prototype

JobExecutionStatus RUNNING

To view the source code for org.apache.spark JobExecutionStatus RUNNING.

Click Source Link

Usage

From source file:org.apache.hadoop.hive.ql.exec.spark.status.LocalSparkJobMonitor.java

License:Apache License

public int startMonitor() {
    boolean running = false;
    boolean done = false;
    int rc = 0;//from w ww  . ja  v  a  2 s.c o m
    JobExecutionStatus lastState = null;
    Map<String, SparkStageProgress> lastProgressMap = null;

    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_JOB);
    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING);

    long startTime = System.currentTimeMillis();

    while (true) {
        try {
            JobExecutionStatus state = sparkJobStatus.getState();
            if (LOG.isDebugEnabled()) {
                console.printInfo("state = " + state);
            }

            if (state == null) {
                long timeCount = (System.currentTimeMillis() - startTime) / 1000;
                if (timeCount > monitorTimeoutInteval) {
                    console.printError("Job hasn't been submitted after " + timeCount + "s. Aborting it.");
                    console.printError("Status: " + state);
                    running = false;
                    done = true;
                    rc = 2;
                    break;
                }
            } else if (state != lastState || state == JobExecutionStatus.RUNNING) {
                lastState = state;
                Map<String, SparkStageProgress> progressMap = sparkJobStatus.getSparkStageProgress();

                switch (state) {
                case RUNNING:
                    if (!running) {
                        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING);
                        // print job stages.
                        console.printInfo(
                                "\nQuery Hive on Spark job[" + sparkJobStatus.getJobId() + "] stages:");
                        for (int stageId : sparkJobStatus.getStageIds()) {
                            console.printInfo(Integer.toString(stageId));
                        }

                        console.printInfo(
                                "\nStatus: Running (Hive on Spark job[" + sparkJobStatus.getJobId() + "])");
                        running = true;

                        console.printInfo("Job Progress Format\nCurrentTime StageId_StageAttemptId: "
                                + "SucceededTasksCount(+RunningTasksCount-FailedTasksCount)/TotalTasksCount [StageCost]");
                    }

                    printStatus(progressMap, lastProgressMap);
                    lastProgressMap = progressMap;
                    break;
                case SUCCEEDED:
                    printStatus(progressMap, lastProgressMap);
                    lastProgressMap = progressMap;
                    double duration = (System.currentTimeMillis() - startTime) / 1000.0;
                    console.printInfo(
                            "Status: Finished successfully in " + String.format("%.2f seconds", duration));
                    running = false;
                    done = true;
                    break;
                case FAILED:
                    console.printError("Status: Failed");
                    running = false;
                    done = true;
                    rc = 3;
                    break;
                case UNKNOWN:
                    console.printError("Status: Unknown");
                    running = false;
                    done = true;
                    rc = 4;
                    break;
                }
            }
            if (!done) {
                Thread.sleep(checkInterval);
            }
        } catch (Exception e) {
            String msg = " with exception '" + Utilities.getNameMessage(e) + "'";
            msg = "Failed to monitor Job[ " + sparkJobStatus.getJobId() + "]" + msg;

            // Has to use full name to make sure it does not conflict with
            // org.apache.commons.lang.StringUtils
            LOG.error(msg, e);
            console.printError(msg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
            rc = 1;
            done = true;
        } finally {
            if (done) {
                break;
            }
        }
    }

    perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_RUN_JOB);
    return rc;
}

---

From source file:org.apache.hadoop.hive.ql.exec.spark.status.RemoteSparkJobMonitor.java

License:Apache License

@Override
public int startMonitor() {
    boolean running = false;
    boolean done = false;
    int rc = 0;/*from w  w w  .j  a  v a  2s  . c om*/
    Map<String, SparkStageProgress> lastProgressMap = null;

    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_JOB);
    perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING);

    long startTime = System.currentTimeMillis();

    while (true) {
        try {
            JobHandle.State state = sparkJobStatus.getRemoteJobState();
            if (LOG.isDebugEnabled()) {
                console.printInfo("state = " + state);
            }

            switch (state) {
            case SENT:
            case QUEUED:
                long timeCount = (System.currentTimeMillis() - startTime) / 1000;
                if ((timeCount > monitorTimeoutInteval)) {
                    console.printError("Job hasn't been submitted after " + timeCount + "s."
                            + " Aborting it.\nPossible reasons include network issues, "
                            + "errors in remote driver or the cluster has no available resources, etc.\n"
                            + "Please check YARN or Spark driver's logs for further information.");
                    console.printError("Status: " + state);
                    running = false;
                    done = true;
                    rc = 2;
                }
                break;
            case STARTED:
                JobExecutionStatus sparkJobState = sparkJobStatus.getState();
                if (sparkJobState == JobExecutionStatus.RUNNING) {
                    Map<String, SparkStageProgress> progressMap = sparkJobStatus.getSparkStageProgress();
                    if (!running) {
                        perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING);
                        printAppInfo();
                        // print job stages.
                        console.printInfo(
                                "\nQuery Hive on Spark job[" + sparkJobStatus.getJobId() + "] stages:");
                        for (int stageId : sparkJobStatus.getStageIds()) {
                            console.printInfo(Integer.toString(stageId));
                        }

                        console.printInfo(
                                "\nStatus: Running (Hive on Spark job[" + sparkJobStatus.getJobId() + "])");
                        running = true;

                        console.printInfo("Job Progress Format\nCurrentTime StageId_StageAttemptId: "
                                + "SucceededTasksCount(+RunningTasksCount-FailedTasksCount)/TotalTasksCount [StageCost]");
                    }

                    printStatus(progressMap, lastProgressMap);
                    lastProgressMap = progressMap;
                }
                break;
            case SUCCEEDED:
                Map<String, SparkStageProgress> progressMap = sparkJobStatus.getSparkStageProgress();
                printStatus(progressMap, lastProgressMap);
                lastProgressMap = progressMap;
                double duration = (System.currentTimeMillis() - startTime) / 1000.0;
                console.printInfo(
                        "Status: Finished successfully in " + String.format("%.2f seconds", duration));
                running = false;
                done = true;
                break;
            case FAILED:
                console.printError("Status: Failed");
                running = false;
                done = true;
                rc = 3;
                break;
            }

            if (!done) {
                Thread.sleep(checkInterval);
            }
        } catch (Exception e) {
            String msg = " with exception '" + Utilities.getNameMessage(e) + "'";
            msg = "Failed to monitor Job[ " + sparkJobStatus.getJobId() + "]" + msg;

            // Has to use full name to make sure it does not conflict with
            // org.apache.commons.lang.StringUtils
            LOG.error(msg, e);
            console.printError(msg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
            rc = 1;
            done = true;
        } finally {
            if (done) {
                break;
            }
        }
    }

    perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_RUN_JOB);
    return rc;
}

---

From source file:org.apache.sqoop.submission.spark.SparkSubmissionEngine.java

License:Apache License

/**
 * Convert spark specific job status constants to Sqoop job status
 * constants./*  ww  w . jav  a 2  s.com*/
 *
 * @param status Spark job constant
 * @return Equivalent submission status
 */
private SubmissionStatus convertSparkState(JobExecutionStatus status) {
    if (JobExecutionStatus.RUNNING == status) {
        return SubmissionStatus.RUNNING;
    } else if (JobExecutionStatus.FAILED == status) {
        return SubmissionStatus.FAILED;
    } else if (JobExecutionStatus.UNKNOWN == status) {
        return SubmissionStatus.UNKNOWN;
    } else if (JobExecutionStatus.SUCCEEDED == status) {
        return SubmissionStatus.SUCCEEDED;
    }
    throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0004, "Unknown status " + status);
}

---