Example usage for org.apache.hadoop.mapred TIPStatus PENDING

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred TIPStatus PENDING.

Prototype

TIPStatus PENDING

To view the source code for org.apache.hadoop.mapred TIPStatus PENDING.

Click Source Link

Usage

From source file:azkaban.jobtype.MapReduceJobState.java

License:Apache License

public MapReduceJobState(RunningJob runningJob, TaskReport[] mapTaskReport, TaskReport[] reduceTaskReport)
        throws IOException {
    jobId = runningJob.getID().toString();
    jobName = runningJob.getJobName();//from   ww w  .  ja v  a2 s.  c  o m
    trackingURL = runningJob.getTrackingURL();
    isComplete = runningJob.isComplete();
    isSuccessful = runningJob.isSuccessful();
    mapProgress = runningJob.mapProgress();
    reduceProgress = runningJob.reduceProgress();
    failureInfo = runningJob.getFailureInfo();

    totalMappers = mapTaskReport.length;
    totalReducers = reduceTaskReport.length;

    for (TaskReport report : mapTaskReport) {
        if (report.getStartTime() < jobStartTime || jobStartTime == 0L) {
            jobStartTime = report.getStartTime();
        }

        TIPStatus status = report.getCurrentStatus();
        if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) {
            finishedMappersCount++;
        }
    }

    for (TaskReport report : reduceTaskReport) {
        if (jobLastUpdateTime < report.getFinishTime()) {
            jobLastUpdateTime = report.getFinishTime();
        }

        TIPStatus status = report.getCurrentStatus();
        if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) {
            finishedReducersCount++;
        }
    }

    // If not all the reducers are finished.
    if (finishedReducersCount != reduceTaskReport.length || jobLastUpdateTime == 0) {
        jobLastUpdateTime = System.currentTimeMillis();
    }

    counters = runningJob.getCounters();
}

---

From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java

License:Open Source License

/**
 * Gets the task report.//from   w  ww .  ja v  a 2s.  c om
 * 
 * @param taskReports
 *            the task reports
 * @return the task report
 */
private Map<String, Object> getTaskReport(TaskReport[] taskReports) {
    Map<String, Object> taskReportsInfo = new HashMap<String, Object>();
    try {
        LOG.info("Total Task : " + taskReports.length);
        List<Map> taskLists = new ArrayList<Map>();
        // A report on the state of a task.
        if (taskReports != null) {
            int completeTask = 0;
            int failedTask = 0;
            int killedTask = 0;
            int runningTask = 0;
            int pendingTask = 0;
            Map<String, Object[]> diagInfo = new HashMap<String, Object[]>();
            // Iterating over the task reports
            for (TaskReport mtr : taskReports) {
                // Creating an empty map for storing task details
                Map<String, Object> taskReport = new HashMap<String, Object>();
                // The current status of the task
                TIPStatus currentStatus = mtr.getCurrentStatus();
                // Checking for task's current status COMPLETE
                if (currentStatus == TIPStatus.COMPLETE) {
                    completeTask++;
                }
                // Checking for task's current status KILLED
                if (currentStatus == TIPStatus.KILLED) {
                    killedTask++;
                }
                // Checking for task's current status RUNNING
                if (currentStatus == TIPStatus.RUNNING) {
                    runningTask++;
                }
                // Checking for task's current status PENDING
                if (currentStatus == TIPStatus.PENDING) {
                    pendingTask++;
                }
                // The id of the task.
                TaskID taskId = mtr.getTaskID();
                float progress = mtr.getProgress();
                // The most recent state
                String state = mtr.getState();

                // Putting value in a map
                taskReport.put("taskId", taskId.toString());
                taskReport.put("successfulTaskAttemp", mtr.getSuccessfulTaskAttempt().toString());
                taskReport.put("startTime", mtr.getStartTime());
                taskReport.put("finishTime", mtr.getFinishTime());
                taskReport.put("progress", progress * 100);
                taskReport.put("state", state);
                taskReport.put("currentStatus", currentStatus);
                Counters counters = mtr.getCounters();
                List countersList = new ArrayList();
                for (Group group : counters) {
                    Map<String, Object> counterMap = new HashMap<String, Object>();
                    counterMap.put("name", group.getDisplayName());
                    List subCounters = new ArrayList();
                    for (Counter counter : group) {
                        Map subCounter = new HashMap();
                        subCounter.put("name", counter.getDisplayName());
                        subCounter.put("value", counter.getCounter());
                        subCounters.add(subCounter);
                    }
                    counterMap.put("subCounters", subCounters);
                    countersList.add(counterMap);
                }
                taskReport.put("counters", countersList);
                taskLists.add(taskReport);
                // A list of error messages.
                String[] diagnostics = mtr.getDiagnostics();
                if (diagnostics != null) {
                    int count = 0;
                    // Iterating over the list of error messages
                    for (String di : diagnostics) {
                        Object[] diagStatus = new Object[2];
                        diagStatus[0] = taskId;
                        diagStatus[1] = di;
                        diagInfo.put(taskId + "_" + count, diagStatus);
                        count++;
                    }
                }
            }
            // Putting value in a map
            taskReportsInfo.put("completedTask", completeTask);
            taskReportsInfo.put("pendingTask", pendingTask);
            taskReportsInfo.put("killedTask", killedTask);
            taskReportsInfo.put("runningTask", runningTask);
            taskReportsInfo.put("failedTask", failedTask);
            taskReportsInfo.put("failedOrKilledTask", failedTask);
            taskReportsInfo.put("diagInfo", diagInfo);
            taskReportsInfo.put("tasks", taskLists);
        }
    } catch (Exception e) {
        HadoopUtils.addAndLogError(this.LOG, this.clusterConfig, "Could not get task report",
                Constant.Component.Name.HADOOP, e);
    }
    return taskReportsInfo;
}

---