List of usage examples for org.apache.hadoop.mapred RunningJob getJobName
public String getJobName();
From source file:azkaban.jobtype.MapReduceJobState.java
License:Apache License
public MapReduceJobState(RunningJob runningJob, TaskReport[] mapTaskReport, TaskReport[] reduceTaskReport) throws IOException { jobId = runningJob.getID().toString(); jobName = runningJob.getJobName(); trackingURL = runningJob.getTrackingURL(); isComplete = runningJob.isComplete(); isSuccessful = runningJob.isSuccessful(); mapProgress = runningJob.mapProgress(); reduceProgress = runningJob.reduceProgress(); failureInfo = runningJob.getFailureInfo(); totalMappers = mapTaskReport.length; totalReducers = reduceTaskReport.length; for (TaskReport report : mapTaskReport) { if (report.getStartTime() < jobStartTime || jobStartTime == 0L) { jobStartTime = report.getStartTime(); }/*from w ww . j av a 2 s . co m*/ TIPStatus status = report.getCurrentStatus(); if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) { finishedMappersCount++; } } for (TaskReport report : reduceTaskReport) { if (jobLastUpdateTime < report.getFinishTime()) { jobLastUpdateTime = report.getFinishTime(); } TIPStatus status = report.getCurrentStatus(); if (status != TIPStatus.PENDING && status != TIPStatus.RUNNING) { finishedReducersCount++; } } // If not all the reducers are finished. if (finishedReducersCount != reduceTaskReport.length || jobLastUpdateTime == 0) { jobLastUpdateTime = System.currentTimeMillis(); } counters = runningJob.getCounters(); }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.JobLayer.java
License:Apache License
public String getJobStatisticsByJobId(String jobTracker, String jobId) throws IOException { StringBuilder jobStatistics = new StringBuilder(); JobClient client = prepareJobClient(jobTracker); JobID id = convertToJobId(jobId);/*from w w w .j a va 2 s .c o m*/ RunningJob job = client.getJob(id); double mapProgress = job.mapProgress() * 100; double reduceProgress = job.reduceProgress() * 100; String mapPercentage = Double.toString(mapProgress) + "%"; String reducePercentage = Double.toString(reduceProgress) + "%"; jobStatistics.append("<job id='").append(jobId).append("'" + " name='").append(job.getJobName()) .append("'>\n"); jobStatistics.append(" <mapProgress>").append(mapPercentage).append("</mapProgress>\n"); jobStatistics.append(" <reduceProgress>").append(reducePercentage).append("</reduceProgress>\n"); jobStatistics.append(" <complete>").append(job.isComplete()).append("</complete>\n"); jobStatistics.append(" <successful>").append(job.isSuccessful()).append("</successful>\n"); jobStatistics.append(" <url>").append(job.getTrackingURL()).append("</url>\n"); jobStatistics.append("</job>"); return jobStatistics.toString(); }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer.java
License:Apache License
public String getTaskLevelCountersByJobId(String jobTracker, String jobId) throws IOException { StringBuilder taskCounters = new StringBuilder(); JobID id = this.convertToJobId(jobId); JobClient client = this.prepareJobClient(jobTracker); RunningJob job = client.getJob(id); TaskReport[] mapTaskReports = client.getMapTaskReports(id); TaskReport[] reduceTaskReports = client.getReduceTaskReports(id); taskCounters.append("<job id='").append(jobId).append("' name='").append(job.getJobName()).append("'>\n"); taskCounters.append(" <mapTasks>\n"); for (TaskReport mapTaskReport : mapTaskReports) { taskCounters.append(" <task id='").append(mapTaskReport.getTaskID().toString()).append("'\n"); taskCounters.append(" <counters>\n"); Counters counter = mapTaskReport.getCounters(); Iterator<Group> iter = counter.iterator(); while (iter.hasNext()) { Group group = iter.next(); Iterator<Counter> cIter = group.iterator(); while (cIter.hasNext()) { Counter c = cIter.next(); taskCounters.append(" <counter name='").append(c.getDisplayName()).append("' value='") .append(c.getValue()).append("'>\n"); }// w ww. jav a2s . com } taskCounters.append(" </counters>\n"); taskCounters.append(" </task>\n"); } taskCounters.append(" </mapTasks>\n"); taskCounters.append(" <reduceTasks>\n"); for (TaskReport reduceTaskReport : reduceTaskReports) { taskCounters.append(" <task id='").append(reduceTaskReport.getTaskID().toString()).append("'\n"); taskCounters.append(" <counters>\n"); Counters counter = reduceTaskReport.getCounters(); Iterator<Group> iter = counter.iterator(); while (iter.hasNext()) { Group group = iter.next(); Iterator<Counter> cIter = group.iterator(); while (cIter.hasNext()) { Counter c = cIter.next(); taskCounters.append(" <counter name='").append(c.getDisplayName()).append("' value='") .append(c.getValue()).append("'>\n"); } } taskCounters.append(" </counters>\n"); taskCounters.append(" </task>\n"); } taskCounters.append(" </reduceTasks>\n"); taskCounters.append("</job>"); return taskCounters.toString(); }
From source file:com.atlantbh.jmeter.plugins.hadooputilities.jobstatistics.TaskLayer.java
License:Apache License
public String getTaskStatisticsByJobId(String jobTracker, String jobId) throws IOException { StringBuilder taskStatistics = new StringBuilder(); long taskDuration; String duration;//from w w w .ja v a 2 s. c om JobID id = this.convertToJobId(jobId); JobClient client = this.prepareJobClient(jobTracker); RunningJob job = client.getJob(id); TaskReport[] mapTaskReports = client.getMapTaskReports(id); TaskReport[] reduceTaskReports = client.getReduceTaskReports(id); taskStatistics.append("<job id='").append(jobId).append("' name='").append(job.getJobName()).append("'>\n"); taskStatistics.append(" <mapTasks>\n"); for (TaskReport mapTaskReport : mapTaskReports) { taskDuration = mapTaskReport.getFinishTime() - mapTaskReport.getStartTime(); if (taskDuration < 0) { duration = "N/A"; } else { duration = String.valueOf(taskDuration); } double progress = mapTaskReport.getProgress() * 100; String taskProgress = Double.toString(progress) + "%"; taskStatistics.append(" <task id='").append(mapTaskReport.getTaskID().toString()).append("'\n"); taskStatistics.append(" <progress>").append(taskProgress).append("</progress>\n"); taskStatistics.append(" <duration>").append(duration).append("</duration>\n"); taskStatistics.append(" <status>").append(mapTaskReport.getCurrentStatus().toString()) .append("</status>\n"); taskStatistics.append(" </task>\n"); } taskStatistics.append(" </mapTasks>\n"); taskStatistics.append(" <reduceTasks>\n"); for (TaskReport reduceTaskReport : reduceTaskReports) { taskDuration = reduceTaskReport.getFinishTime() - reduceTaskReport.getStartTime(); if (taskDuration < 0) { duration = "N/A"; } else { duration = String.valueOf(taskDuration); } double progress = reduceTaskReport.getProgress() * 100; String taskProgress = Double.toString(progress) + "%"; taskStatistics.append(" <task id='").append(reduceTaskReport.getTaskID().toString()).append("'\n"); taskStatistics.append(" <progress>").append(taskProgress).append("</progress>\n"); taskStatistics.append(" <duration>").append(duration).append("</duration>\n"); taskStatistics.append(" <status>").append(reduceTaskReport.getCurrentStatus().toString()) .append("</status>\n"); taskStatistics.append(" </task>\n"); } taskStatistics.append(" </reduceTasks>\n"); taskStatistics.append("</job>"); return taskStatistics.toString(); }
From source file:com.ibm.jaql.lang.expr.hadoop.Util.java
License:Apache License
public static void submitJob(JsonString submitClassName, JobConf conf) throws Exception { JobClient jc = new JobClient(conf); RunningJob rj = jc.submitJob(conf); String sc = JsonUtil.printToString(submitClassName); // log to status that a MR job is starting mrStatusStart(sc);/*from ww w . j av a 2 s .co m*/ // log to status vital MR job information mrStatusInfo(sc, JsonUtil.printToString(new JsonString(rj.getID().toString())), JsonUtil.printToString(new JsonString(rj.getJobName())), JsonUtil.printToString(new JsonString(rj.getTrackingURL()))); //STATUS_LOG.info("MAP-REDUCE INFO: " + rj.getID() + "," + rj.getJobName() + "," + rj.getTrackingURL()); boolean failed = false; try { if (!jc.monitorAndPrintJob(conf, rj)) { LOG.error(new IOException("Job failed!")); failed = true; //throw new IOException("Job failed!"); } } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } try { if (rj.isSuccessful()) { logAllTaskSyslogs(rj, true); } else { logAllTaskSyslogs(rj, false); } } catch (Throwable t) { // log it, but do not stop the world for this LOG.error(t); } // log to status that a MR job is stopping mrStatusStop(sc); // if the job failed, then throw an exception if (failed) { throw new IOException("Job failed!"); } }
From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * @param jobClient/* w w w.ja v a2 s . c o m*/ * @param jobSts * @return * @throws IOException */ private Map<String, Object> getJobReport(JobStatus jobSts) throws IOException { // Creating an empty map for storing job information Map<String, Object> jobReport = new HashMap<String, Object>(); // Returns the jobid of the Job org.apache.hadoop.mapred.JobID jobId = jobSts.getJobID(); // Get an RunningJob object to track an ongoing Map-Reduce // job. RunningJob job = jobClient.getJob(jobId); String jobName = ""; if (job != null) { // Get the name of the job. jobName = job.getJobName(); } // Percentage of progress in maps float mapProgress = jobSts.mapProgress() * 100; // Percentage of progress in reduce float reduceProgress = jobSts.reduceProgress() * 100; int mapTotal = 0; int reduceTotal = 0; int mapComp = 0; int reduceComp = 0; // Count for Map and Reduce Complete try { // Get the information of the current state of the map // tasks of a job TaskReport[] mapTaskReports = jobClient.getMapTaskReports(jobId); // Get the total map mapTotal = mapTaskReports.length; // Iterating over the map tasks for (TaskReport taskReport : mapTaskReports) { // The current state of a map TaskInProgress as seen // by the JobTracker. TIPStatus currentStatus = taskReport.getCurrentStatus(); if (currentStatus == TIPStatus.COMPLETE) { mapComp++; } } // Get the information of the current state of the // reduce tasks of a job. TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobId); // Get the total reduce reduceTotal = reduceTaskReport.length; // Iterating over the reduce tasks for (TaskReport taskReport : reduceTaskReport) { // The current state of a reduce TaskInProgress as // seen by the JobTracker. TIPStatus currentStatus = taskReport.getCurrentStatus(); if (currentStatus == TIPStatus.COMPLETE) { reduceComp++; } } } catch (Exception e) { LOG.error(e.getMessage(), e); } // Percentage of progress in setup float setupProgress = jobSts.setupProgress() * 100; // The progress made on cleanup float cleanupProgress = jobSts.cleanupProgress() * 100; // gets any available info on the reason of failure of the // job..Returns the diagnostic information on why a job // might have failed. String failureInfo = jobSts.getFailureInfo(); // Putting Job Sttaus information in map jobReport.put("jobId", jobId.toString()); jobReport.put("jobName", jobName); jobReport.put("jobPriority", jobSts.getJobPriority().toString()); jobReport.put("jobStartTime", jobSts.getStartTime()); jobReport.put("userName", jobSts.getUsername()); jobReport.put("jobComplete", jobSts.isJobComplete()); jobReport.put("mapProgress", mapProgress); jobReport.put("reduceProgress", reduceProgress); jobReport.put("mapTotal", mapTotal); jobReport.put("reduceTotal", reduceTotal); jobReport.put("mapCompleted", mapComp); jobReport.put("reduceCompleted", reduceComp); jobReport.put("setupProgress", setupProgress); jobReport.put("cleanupProgress", cleanupProgress); jobReport.put("schedulingInfo", jobSts.getSchedulingInfo()); jobReport.put("jobState", JobStatus.getJobRunState(jobSts.getRunState())); jobReport.put("failureInfo", failureInfo); jobReport.put("jobFile", job.getJobFile()); jobReport.put("trackingURL", job.getTrackingURL()); jobReport.putAll(getDetailedJobReport(jobId)); return jobReport; }
From source file:com.liveramp.cascading_ext.counters.Counters.java
License:Apache License
public static String prettyCountersString(RunningJob job) { StringBuilder builder = new StringBuilder("\n").append(StringUtils.repeat("=", 90)).append("\n"); builder.append("Counters for job ").append(job.getJobName()).append("\n"); for (Counter counter : getStatsFromRunningJob(job, null)) { if (counter.getValue() != null && counter.getValue() > 0) { builder.append(" ").append(counter).append("\n"); }/*w ww. j av a 2 s.com*/ } builder.append(StringUtils.repeat("=", 90)).append("\n"); return builder.toString(); }
From source file:com.netflix.lipstick.pigtolipstick.BasicP2LClient.java
License:Apache License
/** * Build a P2jJobStatus object for the map/reduce job with id jobId. * * @param jobId the id of the map/reduce job * @return the newly created P2jJobStatus *//*from w w w . ja v a 2 s .co m*/ @SuppressWarnings("deprecation") protected P2jJobStatus buildJobStatusMap(String jobId) { JobClient jobClient = PigStats.get().getJobClient(); P2jJobStatus js = jobIdToJobStatusMap.get(jobId); try { RunningJob rj = jobClient.getJob(jobId); if (rj == null) { LOG.warn("Couldn't find job status for jobId=" + jobId); return js; } JobID jobID = rj.getID(); Counters counters = rj.getCounters(); Map<String, P2jCounters> cMap = Maps.newHashMap(); for (Group g : counters) { P2jCounters countersObj = new P2jCounters(); cMap.put(g.getDisplayName(), countersObj); for (Counter c : g) { countersObj.getCounters().put(c.getDisplayName(), c.getValue()); } } js.setCounters(cMap); TaskReport[] mapTaskReport = jobClient.getMapTaskReports(jobID); TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobID); js.setJobName(rj.getJobName()); js.setTrackingUrl(rj.getTrackingURL()); js.setIsComplete(rj.isComplete()); js.setIsSuccessful(rj.isSuccessful()); js.setMapProgress(rj.mapProgress()); js.setReduceProgress(rj.reduceProgress()); js.setTotalMappers(mapTaskReport.length); js.setTotalReducers(reduceTaskReport.length); return js; } catch (IOException e) { LOG.error("Error getting job info.", e); } return null; }
From source file:com.twitter.ambrose.hive.AmbroseHiveUtil.java
License:Apache License
/** * Returns the nodeId of the given running job <br> * Example: Stage-1_[queryId]/*w w w. j a v a 2 s. c om*/ * * @param conf * @param runningJob * @return */ public static String getNodeIdFromJob(Configuration conf, RunningJob runningJob) { return getNodeIdFromJobName(conf, runningJob.getJobName()); }
From source file:dataload.LogFetchJobTracker.java
License:Apache License
/** * This is the method to get all of the jobParameters for a given job * @param id//ww w . j ava 2 s . co m * @throws IOException * @throws SQLException */ public void getJobParameters(JobID id) throws IOException, SQLException { RunningJob job = client.getJob(id); Counters c = job.getCounters(); Iterator<Counters.Group> itrG = c.iterator(); PreparedStatement prepStatement = connection.prepareStatement( "INSERT INTO jobParameters VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); for (int i = 1; i < 31; i++) { prepStatement.setLong(i, 0); } prepStatement.setString(31, job.getID().toString()); prepStatement.setString(32, job.getJobName()); prepStatement.setLong(33, totalTime); while (itrG.hasNext()) { Iterator<Counters.Counter> itrC = itrG.next().iterator(); while (itrC.hasNext()) { Counters.Counter counter = itrC.next(); if (mapJob.get(counter.getName()) != null) { prepStatement.setLong(mapJob.get(counter.getName()), counter.getCounter()); } } } prepStatement.executeUpdate(); }