List of usage examples for org.apache.hadoop.mapred JobStatus getUsername
public synchronized String getUsername()
From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * @param jobClient/*from w w w . jav a 2 s.c om*/ * @param jobSts * @return * @throws IOException */ private Map<String, Object> getJobReport(JobStatus jobSts) throws IOException { // Creating an empty map for storing job information Map<String, Object> jobReport = new HashMap<String, Object>(); // Returns the jobid of the Job org.apache.hadoop.mapred.JobID jobId = jobSts.getJobID(); // Get an RunningJob object to track an ongoing Map-Reduce // job. RunningJob job = jobClient.getJob(jobId); String jobName = ""; if (job != null) { // Get the name of the job. jobName = job.getJobName(); } // Percentage of progress in maps float mapProgress = jobSts.mapProgress() * 100; // Percentage of progress in reduce float reduceProgress = jobSts.reduceProgress() * 100; int mapTotal = 0; int reduceTotal = 0; int mapComp = 0; int reduceComp = 0; // Count for Map and Reduce Complete try { // Get the information of the current state of the map // tasks of a job TaskReport[] mapTaskReports = jobClient.getMapTaskReports(jobId); // Get the total map mapTotal = mapTaskReports.length; // Iterating over the map tasks for (TaskReport taskReport : mapTaskReports) { // The current state of a map TaskInProgress as seen // by the JobTracker. TIPStatus currentStatus = taskReport.getCurrentStatus(); if (currentStatus == TIPStatus.COMPLETE) { mapComp++; } } // Get the information of the current state of the // reduce tasks of a job. TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobId); // Get the total reduce reduceTotal = reduceTaskReport.length; // Iterating over the reduce tasks for (TaskReport taskReport : reduceTaskReport) { // The current state of a reduce TaskInProgress as // seen by the JobTracker. TIPStatus currentStatus = taskReport.getCurrentStatus(); if (currentStatus == TIPStatus.COMPLETE) { reduceComp++; } } } catch (Exception e) { LOG.error(e.getMessage(), e); } // Percentage of progress in setup float setupProgress = jobSts.setupProgress() * 100; // The progress made on cleanup float cleanupProgress = jobSts.cleanupProgress() * 100; // gets any available info on the reason of failure of the // job..Returns the diagnostic information on why a job // might have failed. String failureInfo = jobSts.getFailureInfo(); // Putting Job Sttaus information in map jobReport.put("jobId", jobId.toString()); jobReport.put("jobName", jobName); jobReport.put("jobPriority", jobSts.getJobPriority().toString()); jobReport.put("jobStartTime", jobSts.getStartTime()); jobReport.put("userName", jobSts.getUsername()); jobReport.put("jobComplete", jobSts.isJobComplete()); jobReport.put("mapProgress", mapProgress); jobReport.put("reduceProgress", reduceProgress); jobReport.put("mapTotal", mapTotal); jobReport.put("reduceTotal", reduceTotal); jobReport.put("mapCompleted", mapComp); jobReport.put("reduceCompleted", reduceComp); jobReport.put("setupProgress", setupProgress); jobReport.put("cleanupProgress", cleanupProgress); jobReport.put("schedulingInfo", jobSts.getSchedulingInfo()); jobReport.put("jobState", JobStatus.getJobRunState(jobSts.getRunState())); jobReport.put("failureInfo", failureInfo); jobReport.put("jobFile", job.getJobFile()); jobReport.put("trackingURL", job.getTrackingURL()); jobReport.putAll(getDetailedJobReport(jobId)); return jobReport; }
From source file:dataload.LogFetchJobTracker.java
License:Apache License
/** * This method is the method run by the timer */// w w w .j a va2s. com public void run() { try { for (JobStatus stat : client.getAllJobs()) { if (stat.getUsername().equals("data_svc") && !jobsSeen.contains(stat.getJobID().toString())) { totalTime = 0; makeTable(stat.getJobID()); writeMapAndReduceCounters(stat.getJobID()); getJobParameters(stat.getJobID()); jobsSeen.add(stat.getJobID().toString()); } } } catch (SQLException e) { e.printStackTrace(); System.err.println("Error: " + e.getMessage()); } catch (IOException e) { e.printStackTrace(); System.err.println("Error: " + e.getMessage()); } }
From source file:eu.scape_project.tb.hadoopjobtracker.HadoobJobTrackerClient.java
License:Apache License
public HDJobStatus[] UUIDJobs(String UUID) { JobStatus[] allHadoopJobs = null;/* ww w . j ava 2 s. c o m*/ try { allHadoopJobs = myJobClient.getAllJobs(); } catch (IOException ex) { logger.error("Error retrieving ALL jobs from JobTracker. ERR: " + ex.getMessage()); } int numberOfJobs = allHadoopJobs.length; logger.info("Number of ALL jobs on the cluster: " + numberOfJobs); logger.info("Searching for jobs containing the UUID: '" + UUID + "'"); ArrayList<HDJobStatus> allUUIDJobs = new ArrayList<HDJobStatus>(); if (numberOfJobs > 0) { for (JobStatus singleJobStatus : allHadoopJobs) { JobID singleJobID = singleJobStatus.getJobID(); String singleJobName = "N/A"; try { singleJobName = getJobName(myJobClient, singleJobID); } catch (IOException ex) { logger.error("Error retrieving jobName for job " + singleJobID + ". ERR: " + ex.getMessage()); } if ((singleJobName.toLowerCase().indexOf(UUID.toLowerCase()) >= 0) || (UUID.equals(""))) { HDJobStatus newJobStatus = new HDJobStatus(); newJobStatus.setJobID(singleJobStatus.getJobID().toString()); newJobStatus.setJobIsComplete(singleJobStatus.isJobComplete()); newJobStatus.setJobUserName(singleJobStatus.getUsername()); newJobStatus.setJobFailureInfo(singleJobStatus.getFailureInfo()); newJobStatus.setJobName(singleJobName); newJobStatus.setJobMapProgress(Math.round(singleJobStatus.mapProgress() * 100)); newJobStatus.setJobReduceProgress(Math.round(singleJobStatus.reduceProgress() * 100)); allUUIDJobs.add(newJobStatus); } } } return allUUIDJobs.toArray(new HDJobStatus[allUUIDJobs.size()]); }
From source file:org.apache.hive.hcatalog.templeton.ListDelegator.java
License:Apache License
public List<String> run(String user, boolean showall) throws NotAuthorizedException, BadParam, IOException, InterruptedException { UserGroupInformation ugi = UgiFactory.getUgi(user); WebHCatJTShim tracker = null;//w w w . ja va2s .co m try { tracker = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); ArrayList<String> ids = new ArrayList<String>(); JobStatus[] jobs = tracker.getAllJobs(); if (jobs != null) { for (JobStatus job : jobs) { String id = job.getJobID().toString(); if (showall || user.equals(job.getUsername())) ids.add(id); } } return ids; } catch (IllegalStateException e) { throw new BadParam(e.getMessage()); } finally { if (tracker != null) tracker.close(); } }
From source file:org.estado.core.JobStatusChecker.java
License:Apache License
public void checkStatus() { List<org.estado.spi.JobStatus> jobStatusList = new ArrayList<org.estado.spi.JobStatus>(); try {//from ww w . j a v a 2s .c o m Configuration conf = new Configuration(); JobClient client = new JobClient(new JobConf(conf)); JobStatus[] jobStatuses = client.getAllJobs(); showFilter(); int jobCount = 0; for (JobStatus jobStatus : jobStatuses) { Long lastTaskEndTime = 0L; TaskReport[] mapReports = client.getMapTaskReports(jobStatus.getJobID()); for (TaskReport r : mapReports) { if (lastTaskEndTime < r.getFinishTime()) { lastTaskEndTime = r.getFinishTime(); } } TaskReport[] reduceReports = client.getReduceTaskReports(jobStatus.getJobID()); for (TaskReport r : reduceReports) { if (lastTaskEndTime < r.getFinishTime()) { lastTaskEndTime = r.getFinishTime(); } } client.getSetupTaskReports(jobStatus.getJobID()); client.getCleanupTaskReports(jobStatus.getJobID()); String jobId = jobStatus.getJobID().toString(); String jobName = client.getJob(jobStatus.getJobID()).getJobName(); Long startTime = jobStatus.getStartTime(); String user = jobStatus.getUsername(); int mapProgress = (int) (jobStatus.mapProgress() * 100); int reduceProgress = (int) (jobStatus.reduceProgress() * 100); org.estado.spi.JobStatus jobStat = null; ++jobCount; int runState = jobStatus.getRunState(); switch (runState) { case JobStatus.SUCCEEDED: if (filter.contains("s")) { Long duration = lastTaskEndTime - jobStatus.getStartTime(); jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime, lastTaskEndTime, duration, mapProgress, reduceProgress, "completed"); ++sCount; } break; case JobStatus.RUNNING: if (filter.contains("r")) { long duration = System.currentTimeMillis() - jobStatus.getStartTime(); jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime, lastTaskEndTime, duration, mapProgress, reduceProgress, "running"); ++rCount; } break; case JobStatus.FAILED: if (filter.contains("f")) { long duration = lastTaskEndTime - jobStatus.getStartTime(); jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime, lastTaskEndTime, duration, mapProgress, reduceProgress, "failed"); RunningJob job = client.getJob(jobStatus.getJobID()); jobStat.setJobTasks(getTaskDetails(job)); ++fCount; } break; case JobStatus.PREP: if (filter.contains("p")) { jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, null, null, null, 0, 0, "preparing"); ++pCount; } break; case JobStatus.KILLED: if (filter.contains("k")) { long duration = lastTaskEndTime - jobStatus.getStartTime(); jobStat = new org.estado.spi.JobStatus(cluster, jobId, jobName, null, user, startTime, lastTaskEndTime, duration, mapProgress, reduceProgress, "killed"); RunningJob job = client.getJob(jobStatus.getJobID()); jobStat.setJobTasks(getTaskDetails(job)); ++kCount; } break; } jobStatusList.add(jobStat); } //get counters for (org.estado.spi.JobStatus jobStat : jobStatusList) { if (!jobStat.getStatus().equals("preparing")) { List<JobCounterGroup> counterGroups = getJobCounters(jobStat.getJobId()); jobStat.setCounterGroups(counterGroups); //additional data from counters setJobInfo(jobStat); } } //publish to all consumers for (JobStatusConsumer consumer : consumers) { consumer.handle(jobStatusList); } showJobCounts(); } catch (Exception ex) { System.out.println("Jobs status checker failed" + ex.getMessage()); } }