List of usage examples for org.apache.hadoop.mapred TIPStatus RUNNING
TIPStatus RUNNING
To view the source code for org.apache.hadoop.mapred TIPStatus RUNNING.
Click Source Link
From source file:azkaban.jobtype.MapReduceJobState.java
License:Apache License
public MapReduceJobState(RunningJob runningJob, TaskReport[] mapTaskReport, TaskReport[] reduceTaskReport) throws IOException { jobId = runningJob.getID().toString(); jobName = runningJob.getJobName();// w ww. j a v a 2 s. com trackingURL = runningJob.getTrackingURL(); isComplete = runningJob.isComplete(); isSuccessful = runningJob.isSuccessful(); mapProgress = runningJob.mapProgress(); reduceProgress = runningJob.reduceProgress(); failureInfo = runningJob.getFailureInfo(); totalMappers = mapTaskReport.length; totalReducers = reduceTaskReport.length; for (TaskReport report : mapTaskReport) { if (report.getStartTime() < jobStartTime || jobStartTime == 0L) { jobStartTime = report.getStartTime(); } TIPStatus status = report.getCurrentStatus(); if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) { finishedMappersCount++; } } for (TaskReport report : reduceTaskReport) { if (jobLastUpdateTime < report.getFinishTime()) { jobLastUpdateTime = report.getFinishTime(); } TIPStatus status = report.getCurrentStatus(); if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) { finishedReducersCount++; } } // If not all the reducers are finished. if (finishedReducersCount != reduceTaskReport.length || jobLastUpdateTime == 0) { jobLastUpdateTime = System.currentTimeMillis(); } counters = runningJob.getCounters(); }
From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * Gets the task report./*ww w . j a va 2s .co m*/ * * @param taskReports * the task reports * @return the task report */ private Map<String, Object> getTaskReport(TaskReport[] taskReports) { Map<String, Object> taskReportsInfo = new HashMap<String, Object>(); try { LOG.info("Total Task : " + taskReports.length); List<Map> taskLists = new ArrayList<Map>(); // A report on the state of a task. if (taskReports != null) { int completeTask = 0; int failedTask = 0; int killedTask = 0; int runningTask = 0; int pendingTask = 0; Map<String, Object[]> diagInfo = new HashMap<String, Object[]>(); // Iterating over the task reports for (TaskReport mtr : taskReports) { // Creating an empty map for storing task details Map<String, Object> taskReport = new HashMap<String, Object>(); // The current status of the task TIPStatus currentStatus = mtr.getCurrentStatus(); // Checking for task's current status COMPLETE if (currentStatus == TIPStatus.COMPLETE) { completeTask++; } // Checking for task's current status KILLED if (currentStatus == TIPStatus.KILLED) { killedTask++; } // Checking for task's current status RUNNING if (currentStatus == TIPStatus.RUNNING) { runningTask++; } // Checking for task's current status PENDING if (currentStatus == TIPStatus.PENDING) { pendingTask++; } // The id of the task. TaskID taskId = mtr.getTaskID(); float progress = mtr.getProgress(); // The most recent state String state = mtr.getState(); // Putting value in a map taskReport.put("taskId", taskId.toString()); taskReport.put("successfulTaskAttemp", mtr.getSuccessfulTaskAttempt().toString()); taskReport.put("startTime", mtr.getStartTime()); taskReport.put("finishTime", mtr.getFinishTime()); taskReport.put("progress", progress * 100); taskReport.put("state", state); taskReport.put("currentStatus", currentStatus); Counters counters = mtr.getCounters(); List countersList = new ArrayList(); for (Group group : counters) { Map<String, Object> counterMap = new HashMap<String, Object>(); counterMap.put("name", group.getDisplayName()); List subCounters = new ArrayList(); for (Counter counter : group) { Map subCounter = new HashMap(); subCounter.put("name", counter.getDisplayName()); subCounter.put("value", counter.getCounter()); subCounters.add(subCounter); } counterMap.put("subCounters", subCounters); countersList.add(counterMap); } taskReport.put("counters", countersList); taskLists.add(taskReport); // A list of error messages. String[] diagnostics = mtr.getDiagnostics(); if (diagnostics != null) { int count = 0; // Iterating over the list of error messages for (String di : diagnostics) { Object[] diagStatus = new Object[2]; diagStatus[0] = taskId; diagStatus[1] = di; diagInfo.put(taskId + "_" + count, diagStatus); count++; } } } // Putting value in a map taskReportsInfo.put("completedTask", completeTask); taskReportsInfo.put("pendingTask", pendingTask); taskReportsInfo.put("killedTask", killedTask); taskReportsInfo.put("runningTask", runningTask); taskReportsInfo.put("failedTask", failedTask); taskReportsInfo.put("failedOrKilledTask", failedTask); taskReportsInfo.put("diagInfo", diagInfo); taskReportsInfo.put("tasks", taskLists); } } catch (Exception e) { HadoopUtils.addAndLogError(this.LOG, this.clusterConfig, "Could not get task report", Constant.Component.Name.HADOOP, e); } return taskReportsInfo; }
From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java
License:Apache License
@Test(timeout = 30000) public void testMapTasks() throws Exception { killCounter = 0;//from ww w. ja v a2 s .co m final String pName = HadoopJobMonitorConfiguration.MAP_MAX_RUNTIME_MIN; final boolean passCheck = true, killed = true, dryRun = true, enforce = true; testTask(TaskType.MAP, pName, 5, 10, enforce, !dryRun, TIPStatus.RUNNING, passCheck, !killed); testTask(TaskType.MAP, pName, 15, 10, enforce, !dryRun, TIPStatus.FAILED, passCheck, !killed); testTask(TaskType.MAP, pName, 15, 10, enforce, !dryRun, TIPStatus.RUNNING, !passCheck, killed); testTask(TaskType.MAP, pName, 15, 10, !enforce, !dryRun, TIPStatus.RUNNING, !passCheck, !killed); testTask(TaskType.MAP, pName, 15, 10, !enforce, dryRun, TIPStatus.RUNNING, !passCheck, !killed); testTask(TaskType.MAP, pName, 15, 10, enforce, dryRun, TIPStatus.RUNNING, !passCheck, !killed); }
From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java
License:Apache License
@Test public void testReduceTasks() throws Exception { killCounter = 0;/*from w w w . j a v a 2 s. co m*/ final String pName = HadoopJobMonitorConfiguration.REDUCE_MAX_RUNTIME_MIN; final boolean passCheck = true, killed = true, dryRun = true, enforce = true; testTask(TaskType.REDUCE, pName, 5, 10, enforce, !dryRun, TIPStatus.RUNNING, passCheck, !killed); testTask(TaskType.REDUCE, pName, 5, 10, 0.01f, enforce, !dryRun, TIPStatus.RUNNING, passCheck, !killed); testTask(TaskType.REDUCE, pName, 15, 10, enforce, !dryRun, TIPStatus.FAILED, passCheck, !killed); testTask(TaskType.REDUCE, pName, 15, 10, enforce, !dryRun, TIPStatus.RUNNING, !passCheck, killed); testTask(TaskType.REDUCE, pName, 15, 10, !enforce, !dryRun, TIPStatus.RUNNING, !passCheck, !killed); testTask(TaskType.REDUCE, pName, 15, 10, !enforce, dryRun, TIPStatus.RUNNING, !passCheck, !killed); testTask(TaskType.REDUCE, pName, 15, 10, enforce, dryRun, TIPStatus.RUNNING, !passCheck, !killed); }
From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java
License:Apache License
public void testProgress(TaskType taskType, String pName) throws Exception { killCounter = 0;/*from ww w . j ava 2 s . c om*/ final boolean passCheck = true, killed = true, dryRun = true, enforce = true; float prevProgress = 0.2f; taskProgressCache.put(org.apache.hadoop.mapred.TaskID.downgrade(taskId), new Progress(prevProgress, now - 4 * MIN)); attemptProgressCache.put(taskAttemptId, new Progress(prevProgress, now - 4 * MIN)); //from now -4 until now expected progress is 0.4f, and threshold is set to 0.2f testTask(taskType, pName, 5, 10, prevProgress + 0.01f, enforce, !dryRun, TIPStatus.RUNNING, !passCheck, killed); taskProgressCache.clear(); attemptProgressCache.clear(); taskProgressCache.put(org.apache.hadoop.mapred.TaskID.downgrade(taskId), new Progress(prevProgress, now - 4 * MIN)); attemptProgressCache.put(taskAttemptId, new Progress(prevProgress, now - 4 * MIN)); testTask(taskType, pName, 5, 10, prevProgress + 0.01f, !enforce, !dryRun, TIPStatus.RUNNING, !passCheck, !killed); taskProgressCache.clear(); attemptProgressCache.clear(); taskProgressCache.put(org.apache.hadoop.mapred.TaskID.downgrade(taskId), new Progress(prevProgress, now - 4 * MIN)); attemptProgressCache.put(taskAttemptId, new Progress(prevProgress, now - 4 * MIN)); testTask(taskType, pName, 5, 10, prevProgress + 0.21f, enforce, !dryRun, TIPStatus.RUNNING, passCheck, !killed); }
From source file:com.twitter.hraven.hadoopJobMonitor.policy.DefaultPolicy.java
License:Apache License
/** * check the status of a task//from w ww . ja v a2s .c om * * @param taskType * @param taskReport * @param appConf * @param currTime * @return true if task is well-behaved */ @Override public boolean checkTask(ApplicationReport appReport, TaskType taskType, TaskReport taskReport, AppConfiguraiton appConf, long currTime) { long startTime = taskReport.getStartTime(); long runTime = currTime - startTime; long maxRunTimeMs = appConf.getMaxTaskLenMin(taskType) * 60 * 1000; TIPStatus tStatus = taskReport.getCurrentStatus(); boolean badTask = (tStatus == TIPStatus.RUNNING && runTime > maxRunTimeMs); if (badTask) return !badTask; badTask = !checkProgress(taskReport.getProgress(), appConf.getProgressThreshold(), maxRunTimeMs, taskReport.getTaskID(), currTime); return !badTask; }