Example usage for org.apache.spark SparkStageInfo numFailedTasks

Introduction

In this page you can find the example usage for org.apache.spark SparkStageInfo numFailedTasks.

Prototype

int numFailedTasks();

Source Link

Usage

From source file:org.apache.hadoop.hive.ql.exec.spark.status.impl.LocalSparkJobStatus.java

License:Apache License

@Override
public Map<String, SparkStageProgress> getSparkStageProgress() {
    Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>();
    for (int stageId : getStageIds()) {
        SparkStageInfo sparkStageInfo = getStageInfo(stageId);
        if (sparkStageInfo != null) {
            int runningTaskCount = sparkStageInfo.numActiveTasks();
            int completedTaskCount = sparkStageInfo.numCompletedTasks();
            int failedTaskCount = sparkStageInfo.numFailedTasks();
            int totalTaskCount = sparkStageInfo.numTasks();
            SparkStageProgress sparkStageProgress = new SparkStageProgress(totalTaskCount, completedTaskCount,
                    runningTaskCount, failedTaskCount);
            stageProgresses.put(//from   w  w  w . j av a  2  s .  c  om
                    String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(),
                    sparkStageProgress);
        }
    }
    return stageProgresses;
}

---

From source file:org.apache.hadoop.hive.ql.exec.spark.status.impl.RemoteSparkJobStatus.java

License:Apache License

@Override
public Map<String, SparkStageProgress> getSparkStageProgress() throws HiveException {
    Map<String, SparkStageProgress> stageProgresses = new HashMap<String, SparkStageProgress>();
    for (int stageId : getStageIds()) {
        SparkStageInfo sparkStageInfo = getSparkStageInfo(stageId);
        if (sparkStageInfo != null && sparkStageInfo.name() != null) {
            int runningTaskCount = sparkStageInfo.numActiveTasks();
            int completedTaskCount = sparkStageInfo.numCompletedTasks();
            int failedTaskCount = sparkStageInfo.numFailedTasks();
            int totalTaskCount = sparkStageInfo.numTasks();
            SparkStageProgress sparkStageProgress = new SparkStageProgress(totalTaskCount, completedTaskCount,
                    runningTaskCount, failedTaskCount);
            stageProgresses.put(//from   ww  w .  j a  v a 2s .com
                    String.valueOf(sparkStageInfo.stageId()) + "_" + sparkStageInfo.currentAttemptId(),
                    sparkStageProgress);
        }
    }
    return stageProgresses;
}

---