List of usage examples for org.apache.hadoop.mapred TaskReport getTaskID
public TaskID getTaskID()
From source file:co.cask.cdap.app.mapreduce.MRJobClient.java
License:Apache License
private List<MRTaskInfo> toMRTaskInfos(TaskReport[] taskReports) { List<MRTaskInfo> taskInfos = Lists.newArrayList(); for (TaskReport taskReport : taskReports) { taskInfos.add(new MRTaskInfo(taskReport.getTaskId(), taskReport.getState(), taskReport.getStartTime(), taskReport.getFinishTime(), taskReport.getProgress(), groupToMap(taskReport.getCounters().getGroup(TaskCounter.class.getName())))); }/* w w w .java2 s . co m*/ return taskInfos; }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer.java
License:Apache License
public String getTaskLevelCountersByJobId(String jobTracker, String jobId) throws IOException { StringBuilder taskCounters = new StringBuilder(); JobID id = this.convertToJobId(jobId); JobClient client = this.prepareJobClient(jobTracker); RunningJob job = client.getJob(id);//from www . j a v a 2 s . com TaskReport[] mapTaskReports = client.getMapTaskReports(id); TaskReport[] reduceTaskReports = client.getReduceTaskReports(id); taskCounters.append("<job id='").append(jobId).append("' name='").append(job.getJobName()).append("'>\n"); taskCounters.append(" <mapTasks>\n"); for (TaskReport mapTaskReport : mapTaskReports) { taskCounters.append(" <task id='").append(mapTaskReport.getTaskID().toString()).append("'\n"); taskCounters.append(" <counters>\n"); Counters counter = mapTaskReport.getCounters(); Iterator<Group> iter = counter.iterator(); while (iter.hasNext()) { Group group = iter.next(); Iterator<Counter> cIter = group.iterator(); while (cIter.hasNext()) { Counter c = cIter.next(); taskCounters.append(" <counter name='").append(c.getDisplayName()).append("' value='") .append(c.getValue()).append("'>\n"); } } taskCounters.append(" </counters>\n"); taskCounters.append(" </task>\n"); } taskCounters.append(" </mapTasks>\n"); taskCounters.append(" <reduceTasks>\n"); for (TaskReport reduceTaskReport : reduceTaskReports) { taskCounters.append(" <task id='").append(reduceTaskReport.getTaskID().toString()).append("'\n"); taskCounters.append(" <counters>\n"); Counters counter = reduceTaskReport.getCounters(); Iterator<Group> iter = counter.iterator(); while (iter.hasNext()) { Group group = iter.next(); Iterator<Counter> cIter = group.iterator(); while (cIter.hasNext()) { Counter c = cIter.next(); taskCounters.append(" <counter name='").append(c.getDisplayName()).append("' value='") .append(c.getValue()).append("'>\n"); } } taskCounters.append(" </counters>\n"); taskCounters.append(" </task>\n"); } taskCounters.append(" </reduceTasks>\n"); taskCounters.append("</job>"); return taskCounters.toString(); }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer.java
License:Apache License
public String getTaskStatisticsByJobId(String jobTracker, String jobId) throws IOException { StringBuilder taskStatistics = new StringBuilder(); long taskDuration; String duration;/*from www. j a v a 2 s . c o m*/ JobID id = this.convertToJobId(jobId); JobClient client = this.prepareJobClient(jobTracker); RunningJob job = client.getJob(id); TaskReport[] mapTaskReports = client.getMapTaskReports(id); TaskReport[] reduceTaskReports = client.getReduceTaskReports(id); taskStatistics.append("<job id='").append(jobId).append("' name='").append(job.getJobName()).append("'>\n"); taskStatistics.append(" <mapTasks>\n"); for (TaskReport mapTaskReport : mapTaskReports) { taskDuration = mapTaskReport.getFinishTime() - mapTaskReport.getStartTime(); if (taskDuration < 0) { duration = "N/A"; } else { duration = String.valueOf(taskDuration); } double progress = mapTaskReport.getProgress() * 100; String taskProgress = Double.toString(progress) + "%"; taskStatistics.append(" <task id='").append(mapTaskReport.getTaskID().toString()).append("'\n"); taskStatistics.append(" <progress>").append(taskProgress).append("</progress>\n"); taskStatistics.append(" <duration>").append(duration).append("</duration>\n"); taskStatistics.append(" <status>").append(mapTaskReport.getCurrentStatus().toString()) .append("</status>\n"); taskStatistics.append(" </task>\n"); } taskStatistics.append(" </mapTasks>\n"); taskStatistics.append(" <reduceTasks>\n"); for (TaskReport reduceTaskReport : reduceTaskReports) { taskDuration = reduceTaskReport.getFinishTime() - reduceTaskReport.getStartTime(); if (taskDuration < 0) { duration = "N/A"; } else { duration = String.valueOf(taskDuration); } double progress = reduceTaskReport.getProgress() * 100; String taskProgress = Double.toString(progress) + "%"; taskStatistics.append(" <task id='").append(reduceTaskReport.getTaskID().toString()).append("'\n"); taskStatistics.append(" <progress>").append(taskProgress).append("</progress>\n"); taskStatistics.append(" <duration>").append(duration).append("</duration>\n"); taskStatistics.append(" <status>").append(reduceTaskReport.getCurrentStatus().toString()) .append("</status>\n"); taskStatistics.append(" </task>\n"); } taskStatistics.append(" </reduceTasks>\n"); taskStatistics.append("</job>"); return taskStatistics.toString(); }
From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * Gets the task report./*from ww w .j av a2s . c o m*/ * * @param taskReports * the task reports * @return the task report */ private Map<String, Object> getTaskReport(TaskReport[] taskReports) { Map<String, Object> taskReportsInfo = new HashMap<String, Object>(); try { LOG.info("Total Task : " + taskReports.length); List<Map> taskLists = new ArrayList<Map>(); // A report on the state of a task. if (taskReports != null) { int completeTask = 0; int failedTask = 0; int killedTask = 0; int runningTask = 0; int pendingTask = 0; Map<String, Object[]> diagInfo = new HashMap<String, Object[]>(); // Iterating over the task reports for (TaskReport mtr : taskReports) { // Creating an empty map for storing task details Map<String, Object> taskReport = new HashMap<String, Object>(); // The current status of the task TIPStatus currentStatus = mtr.getCurrentStatus(); // Checking for task's current status COMPLETE if (currentStatus == TIPStatus.COMPLETE) { completeTask++; } // Checking for task's current status KILLED if (currentStatus == TIPStatus.KILLED) { killedTask++; } // Checking for task's current status RUNNING if (currentStatus == TIPStatus.RUNNING) { runningTask++; } // Checking for task's current status PENDING if (currentStatus == TIPStatus.PENDING) { pendingTask++; } // The id of the task. TaskID taskId = mtr.getTaskID(); float progress = mtr.getProgress(); // The most recent state String state = mtr.getState(); // Putting value in a map taskReport.put("taskId", taskId.toString()); taskReport.put("successfulTaskAttemp", mtr.getSuccessfulTaskAttempt().toString()); taskReport.put("startTime", mtr.getStartTime()); taskReport.put("finishTime", mtr.getFinishTime()); taskReport.put("progress", progress * 100); taskReport.put("state", state); taskReport.put("currentStatus", currentStatus); Counters counters = mtr.getCounters(); List countersList = new ArrayList(); for (Group group : counters) { Map<String, Object> counterMap = new HashMap<String, Object>(); counterMap.put("name", group.getDisplayName()); List subCounters = new ArrayList(); for (Counter counter : group) { Map subCounter = new HashMap(); subCounter.put("name", counter.getDisplayName()); subCounter.put("value", counter.getCounter()); subCounters.add(subCounter); } counterMap.put("subCounters", subCounters); countersList.add(counterMap); } taskReport.put("counters", countersList); taskLists.add(taskReport); // A list of error messages. String[] diagnostics = mtr.getDiagnostics(); if (diagnostics != null) { int count = 0; // Iterating over the list of error messages for (String di : diagnostics) { Object[] diagStatus = new Object[2]; diagStatus[0] = taskId; diagStatus[1] = di; diagInfo.put(taskId + "_" + count, diagStatus); count++; } } } // Putting value in a map taskReportsInfo.put("completedTask", completeTask); taskReportsInfo.put("pendingTask", pendingTask); taskReportsInfo.put("killedTask", killedTask); taskReportsInfo.put("runningTask", runningTask); taskReportsInfo.put("failedTask", failedTask); taskReportsInfo.put("failedOrKilledTask", failedTask); taskReportsInfo.put("diagInfo", diagInfo); taskReportsInfo.put("tasks", taskLists); } } catch (Exception e) { HadoopUtils.addAndLogError(this.LOG, this.clusterConfig, "Could not get task report", Constant.Component.Name.HADOOP, e); } return taskReportsInfo; }
From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusCheckerTest.java
License:Apache License
public boolean testTask(TaskType taskType, String confParamName, long durationMin, final int MAX_RUN, float progress, boolean enforce, boolean dryRun, TIPStatus status, boolean wellBahaved, boolean killed) throws Exception { setTaskAttemptXML(durationMin * MIN, progress); TaskReport taskReport = mock(TaskReport.class); when(taskReport.getCurrentStatus()).thenReturn(status); Collection<TaskAttemptID> attempts = new ArrayList<TaskAttemptID>(); attempts.add(taskAttemptId);/*from w ww . j a v a2 s .co m*/ when(taskReport.getRunningTaskAttemptIds()).thenReturn(attempts); when(taskReport.getTaskID()).thenReturn(org.apache.hadoop.mapred.TaskID.downgrade(taskId)); when(taskReport.getProgress()).thenReturn(progress); vConf.setBoolean(HadoopJobMonitorConfiguration.DRY_RUN, dryRun); Configuration remoteAppConf = new Configuration(); remoteAppConf.setInt(confParamName, MAX_RUN); remoteAppConf.setBoolean(HadoopJobMonitorConfiguration.enforced(confParamName), enforce); when(taskReport.getStartTime()).thenReturn(now - durationMin * MIN); AppConfiguraiton appConf = new AppConfiguraiton(remoteAppConf, vConf); AppConfCache.getInstance().put(appId, appConf); appStatusChecker.init(); appStatusChecker.loadClientService(); boolean res = appStatusChecker.checkTask(taskType, taskReport, now); if (wellBahaved) assertEquals("Well-bahved task does not pass the check", wellBahaved, res); else assertEquals("Not Well-bahved task passes the check", wellBahaved, res); if (killed) { killCounter++; verify(clientService, times(killCounter)).killTask(any(TaskAttemptID.class), Mockito.anyBoolean()); } else verify(clientService, times(killCounter)).killTask(any(TaskAttemptID.class), Mockito.anyBoolean()); return res; }
From source file:datafu.hourglass.jobs.StagedOutputJob.java
License:Apache License
/** * Writes Hadoop counters and other task statistics to a file in the file system. * //from w ww.java2 s . c o m * @param fs * @throws IOException */ private void writeCounters(final FileSystem fs) throws IOException { final Path actualOutputPath = FileOutputFormat.getOutputPath(this); SimpleDateFormat timestampFormat = new SimpleDateFormat("yyyyMMddHHmmss"); String suffix = timestampFormat.format(new Date()); if (_countersParentPath != null) { if (!fs.exists(_countersParentPath)) { _log.info("Creating counter parent path " + _countersParentPath); fs.mkdirs(_countersParentPath, FsPermission.valueOf("-rwxrwxr-x")); } // make the name as unique as possible in this case because this may be a directory // where other counter files will be dropped _countersPath = new Path(_countersParentPath, ".counters." + suffix); } else { _countersPath = new Path(actualOutputPath, ".counters." + suffix); } _log.info(String.format("Writing counters to %s", _countersPath)); FSDataOutputStream counterStream = fs.create(_countersPath); BufferedOutputStream buffer = new BufferedOutputStream(counterStream, 256 * 1024); OutputStreamWriter writer = new OutputStreamWriter(buffer); for (String groupName : getCounters().getGroupNames()) { for (Counter counter : getCounters().getGroup(groupName)) { writeAndLog(writer, String.format("%s=%d", counter.getName(), counter.getValue())); } } JobID jobID = this.getJobID(); org.apache.hadoop.mapred.JobID oldJobId = new org.apache.hadoop.mapred.JobID(jobID.getJtIdentifier(), jobID.getId()); long minStart = Long.MAX_VALUE; long maxFinish = 0; long setupStart = Long.MAX_VALUE; long cleanupFinish = 0; DescriptiveStatistics mapStats = new DescriptiveStatistics(); DescriptiveStatistics reduceStats = new DescriptiveStatistics(); boolean success = true; JobClient jobClient = new JobClient(this.conf); Map<String, String> taskIdToType = new HashMap<String, String>(); TaskReport[] setupReports = jobClient.getSetupTaskReports(oldJobId); if (setupReports.length > 0) { _log.info("Processing setup reports"); for (TaskReport report : jobClient.getSetupTaskReports(oldJobId)) { taskIdToType.put(report.getTaskID().toString(), "SETUP"); if (report.getStartTime() == 0) { _log.warn("Skipping report with zero start time"); continue; } setupStart = Math.min(setupStart, report.getStartTime()); } } else { _log.error("No setup reports"); } TaskReport[] mapReports = jobClient.getMapTaskReports(oldJobId); if (mapReports.length > 0) { _log.info("Processing map reports"); for (TaskReport report : mapReports) { taskIdToType.put(report.getTaskID().toString(), "MAP"); if (report.getFinishTime() == 0 || report.getStartTime() == 0) { _log.warn("Skipping report with zero start or finish time"); continue; } minStart = Math.min(minStart, report.getStartTime()); mapStats.addValue(report.getFinishTime() - report.getStartTime()); } } else { _log.error("No map reports"); } TaskReport[] reduceReports = jobClient.getReduceTaskReports(oldJobId); if (reduceReports.length > 0) { _log.info("Processing reduce reports"); for (TaskReport report : reduceReports) { taskIdToType.put(report.getTaskID().toString(), "REDUCE"); if (report.getFinishTime() == 0 || report.getStartTime() == 0) { _log.warn("Skipping report with zero start or finish time"); continue; } maxFinish = Math.max(maxFinish, report.getFinishTime()); reduceStats.addValue(report.getFinishTime() - report.getStartTime()); } } else { _log.error("No reduce reports"); } TaskReport[] cleanupReports = jobClient.getCleanupTaskReports(oldJobId); if (cleanupReports.length > 0) { _log.info("Processing cleanup reports"); for (TaskReport report : cleanupReports) { taskIdToType.put(report.getTaskID().toString(), "CLEANUP"); if (report.getFinishTime() == 0) { _log.warn("Skipping report with finish time of zero"); continue; } cleanupFinish = Math.max(cleanupFinish, report.getFinishTime()); } } else { _log.error("No cleanup reports"); } if (minStart == Long.MAX_VALUE) { _log.error("Could not determine map-reduce start time"); success = false; } if (maxFinish == 0) { _log.error("Could not determine map-reduce finish time"); success = false; } if (setupStart == Long.MAX_VALUE) { _log.error("Could not determine setup start time"); success = false; } if (cleanupFinish == 0) { _log.error("Could not determine cleanup finish time"); success = false; } // Collect statistics on successful/failed/killed task attempts, categorized by setup/map/reduce/cleanup. // Unfortunately the job client doesn't have an easier way to get these statistics. Map<String, Integer> attemptStats = new HashMap<String, Integer>(); _log.info("Processing task attempts"); for (TaskCompletionEvent event : getTaskCompletionEvents(jobClient, oldJobId)) { String type = taskIdToType.get(event.getTaskAttemptId().getTaskID().toString()); String status = event.getTaskStatus().toString(); String key = String.format("%s_%s_ATTEMPTS", status, type); if (!attemptStats.containsKey(key)) { attemptStats.put(key, 0); } attemptStats.put(key, attemptStats.get(key) + 1); } if (success) { writeAndLog(writer, String.format("SETUP_START_TIME_MS=%d", setupStart)); writeAndLog(writer, String.format("CLEANUP_FINISH_TIME_MS=%d", cleanupFinish)); writeAndLog(writer, String.format("COMPLETE_WALL_CLOCK_TIME_MS=%d", cleanupFinish - setupStart)); writeAndLog(writer, String.format("MAP_REDUCE_START_TIME_MS=%d", minStart)); writeAndLog(writer, String.format("MAP_REDUCE_FINISH_TIME_MS=%d", maxFinish)); writeAndLog(writer, String.format("MAP_REDUCE_WALL_CLOCK_TIME_MS=%d", maxFinish - minStart)); writeAndLog(writer, String.format("MAP_TOTAL_TASKS=%d", (long) mapStats.getN())); writeAndLog(writer, String.format("MAP_MAX_TIME_MS=%d", (long) mapStats.getMax())); writeAndLog(writer, String.format("MAP_MIN_TIME_MS=%d", (long) mapStats.getMin())); writeAndLog(writer, String.format("MAP_AVG_TIME_MS=%d", (long) mapStats.getMean())); writeAndLog(writer, String.format("MAP_STD_TIME_MS=%d", (long) mapStats.getStandardDeviation())); writeAndLog(writer, String.format("MAP_SUM_TIME_MS=%d", (long) mapStats.getSum())); writeAndLog(writer, String.format("REDUCE_TOTAL_TASKS=%d", (long) reduceStats.getN())); writeAndLog(writer, String.format("REDUCE_MAX_TIME_MS=%d", (long) reduceStats.getMax())); writeAndLog(writer, String.format("REDUCE_MIN_TIME_MS=%d", (long) reduceStats.getMin())); writeAndLog(writer, String.format("REDUCE_AVG_TIME_MS=%d", (long) reduceStats.getMean())); writeAndLog(writer, String.format("REDUCE_STD_TIME_MS=%d", (long) reduceStats.getStandardDeviation())); writeAndLog(writer, String.format("REDUCE_SUM_TIME_MS=%d", (long) reduceStats.getSum())); writeAndLog(writer, String.format("MAP_REDUCE_SUM_TIME_MS=%d", (long) mapStats.getSum() + (long) reduceStats.getSum())); for (Map.Entry<String, Integer> attemptStat : attemptStats.entrySet()) { writeAndLog(writer, String.format("%s=%d", attemptStat.getKey(), attemptStat.getValue())); } } writer.close(); buffer.close(); counterStream.close(); }
From source file:dataload.LogFetchJobTracker.java
License:Apache License
/** * This does the insertion of a given Task Report into the table * @param prepStatement// ww w .j ava 2 s. co m * @param reports * @param id * @throws SQLException */ public void insertTaskIntoTable(PreparedStatement prepStatement, TaskReport[] reports, JobID id) throws SQLException { for (TaskReport rep : reports) { Counters c = rep.getCounters(); Iterator<Counters.Group> itrG = c.iterator(); prepStatement = connection.prepareStatement("INSERT INTO " + id + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); for (int i = 1; i < 24; i++) { prepStatement.setLong(i, 0); } prepStatement.setString(24, rep.getTaskID().toString()); prepStatement.setLong(25, 0); if (!(rep.getFinishTime() == 0) && !(rep.getStartTime() == 0)) { prepStatement.setLong(25, (rep.getFinishTime() - rep.getStartTime()) / 1000); totalTime += (rep.getFinishTime() - rep.getStartTime()); } else { prepStatement.setLong(25, 0); } while (itrG.hasNext()) { Iterator<Counters.Counter> itrC = itrG.next().iterator(); while (itrC.hasNext()) { Counters.Counter counter = itrC.next(); if (mapCounter.get(counter.getName()) != null) { prepStatement.setLong(mapCounter.get(counter.getName()), counter.getCounter()); } } } prepStatement.executeUpdate(); } }
From source file:org.godhuli.rhipe.FileUtils.java
License:Apache License
private REXP TaskReportToRexp(TaskReport t) { REXP.Builder thevals = REXP.newBuilder(); thevals.setRclass(REXP.RClass.LIST); thevals.addRexpValue(FileUtils.buildlistFromOldCounter(t.getCounters(), 0)); thevals.addRexpValue(RObjects.makeStringVector(convertTIP(t.getCurrentStatus()))); thevals.addRexpValue(//from www . j ava2s. c o m RObjects.buildDoubleVector(new double[] { t.getProgress(), t.getStartTime(), t.getFinishTime() })); thevals.addRexpValue(RObjects.makeStringVector(t.getTaskID().toString())); thevals.addRexpValue(RObjects.makeStringVector(t.getSuccessfulTaskAttempt().toString())); return thevals.build(); }