Example usage for org.apache.spark SparkStageInfo stageId

Introduction

In this page you can find the example usage for org.apache.spark SparkStageInfo stageId.

Prototype

int stageId();

Source Link

Usage

From source file:org.apache.hadoop.hive.ql.exec.spark.status.impl.LocalSparkJobStatus.java

License:Apache License

@Override
public Map<String, SparkStageProgress> getSparkStageProgress() {
    Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>();
    for (int stageId : getStageIds()) {
        SparkStageInfo sparkStageInfo = getStageInfo(stageId);
        if (sparkStageInfo != null) {
            int runningTaskCount = sparkStageInfo.numActiveTasks();
            int completedTaskCount = sparkStageInfo.numCompletedTasks();
            int failedTaskCount = sparkStageInfo.numFailedTasks();
            int totalTaskCount = sparkStageInfo.numTasks();
            SparkStageProgress sparkStageProgress = new SparkStageProgress(totalTaskCount, completedTaskCount,
                    runningTaskCount, failedTaskCount);
            stageProgresses.put(//from   w w w .  j  a  v a2 s  .  c  o m
                    String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(),
                    sparkStageProgress);
        }
    }
    return stageProgresses;
}

---

From source file:org.apache.hadoop.hive.ql.exec.spark.status.impl.RemoteSparkJobStatus.java

License:Apache License

@Override
public Map<String, SparkStageProgress> getSparkStageProgress() throws HiveException {
    Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>();
    for (int stageId : getStageIds()) {
        SparkStageInfo sparkStageInfo = getSparkStageInfo(stageId);
        if (sparkStageInfo != null && sparkStageInfo.name() != null) {
            int runningTaskCount = sparkStageInfo.numActiveTasks();
            int completedTaskCount = sparkStageInfo.numCompletedTasks();
            int failedTaskCount = sparkStageInfo.numFailedTasks();
            int totalTaskCount = sparkStageInfo.numTasks();
            SparkStageProgress sparkStageProgress = new SparkStageProgress(totalTaskCount, completedTaskCount,
                    runningTaskCount, failedTaskCount);
            stageProgresses.put(//from w w w.  j a  va2s. c o  m
                    String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(),
                    sparkStageProgress);
        }
    }
    return stageProgresses;
}

---