List of usage examples for org.apache.hadoop.mapred RunningJob getJobFile
public String getJobFile();
From source file:azkaban.jobtype.StatsUtils.java
License:Apache License
public static Properties getJobConf(RunningJob runningJob) { try {/*w w w . j av a2s . c om*/ Path path = new Path(runningJob.getJobFile()); Configuration conf = new Configuration(false); FileSystem fs = FileSystem.get(new Configuration()); InputStream in = fs.open(path); conf.addResource(in); return getJobConf(conf); } catch (FileNotFoundException e) { logger.warn("Job conf not found."); } catch (IOException e) { logger.warn("Error while retrieving job conf: " + e.getMessage()); } return null; }
From source file:cascading.flow.hadoop.HadoopStepStats.java
License:Open Source License
public void captureJobStats() { RunningJob runningJob = getRunningJob(); if (runningJob == null) return;/* w ww. j a va 2 s . com*/ JobConf ranJob = new JobConf(runningJob.getJobFile()); setNumMapTasks(ranJob.getNumMapTasks()); setNumReducerTasks(ranJob.getNumReduceTasks()); }
From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * @param jobClient// w w w . ja v a2s .c o m * @param jobSts * @return * @throws IOException */ private Map<String, Object> getJobReport(JobStatus jobSts) throws IOException { // Creating an empty map for storing job information Map<String, Object> jobReport = new HashMap<String, Object>(); // Returns the jobid of the Job org.apache.hadoop.mapred.JobID jobId = jobSts.getJobID(); // Get an RunningJob object to track an ongoing Map-Reduce // job. RunningJob job = jobClient.getJob(jobId); String jobName = ""; if (job != null) { // Get the name of the job. jobName = job.getJobName(); } // Percentage of progress in maps float mapProgress = jobSts.mapProgress() * 100; // Percentage of progress in reduce float reduceProgress = jobSts.reduceProgress() * 100; int mapTotal = 0; int reduceTotal = 0; int mapComp = 0; int reduceComp = 0; // Count for Map and Reduce Complete try { // Get the information of the current state of the map // tasks of a job TaskReport[] mapTaskReports = jobClient.getMapTaskReports(jobId); // Get the total map mapTotal = mapTaskReports.length; // Iterating over the map tasks for (TaskReport taskReport : mapTaskReports) { // The current state of a map TaskInProgress as seen // by the JobTracker. TIPStatus currentStatus = taskReport.getCurrentStatus(); if (currentStatus == TIPStatus.COMPLETE) { mapComp++; } } // Get the information of the current state of the // reduce tasks of a job. TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobId); // Get the total reduce reduceTotal = reduceTaskReport.length; // Iterating over the reduce tasks for (TaskReport taskReport : reduceTaskReport) { // The current state of a reduce TaskInProgress as // seen by the JobTracker. TIPStatus currentStatus = taskReport.getCurrentStatus(); if (currentStatus == TIPStatus.COMPLETE) { reduceComp++; } } } catch (Exception e) { LOG.error(e.getMessage(), e); } // Percentage of progress in setup float setupProgress = jobSts.setupProgress() * 100; // The progress made on cleanup float cleanupProgress = jobSts.cleanupProgress() * 100; // gets any available info on the reason of failure of the // job..Returns the diagnostic information on why a job // might have failed. String failureInfo = jobSts.getFailureInfo(); // Putting Job Sttaus information in map jobReport.put("jobId", jobId.toString()); jobReport.put("jobName", jobName); jobReport.put("jobPriority", jobSts.getJobPriority().toString()); jobReport.put("jobStartTime", jobSts.getStartTime()); jobReport.put("userName", jobSts.getUsername()); jobReport.put("jobComplete", jobSts.isJobComplete()); jobReport.put("mapProgress", mapProgress); jobReport.put("reduceProgress", reduceProgress); jobReport.put("mapTotal", mapTotal); jobReport.put("reduceTotal", reduceTotal); jobReport.put("mapCompleted", mapComp); jobReport.put("reduceCompleted", reduceComp); jobReport.put("setupProgress", setupProgress); jobReport.put("cleanupProgress", cleanupProgress); jobReport.put("schedulingInfo", jobSts.getSchedulingInfo()); jobReport.put("jobState", JobStatus.getJobRunState(jobSts.getRunState())); jobReport.put("failureInfo", failureInfo); jobReport.put("jobFile", job.getJobFile()); jobReport.put("trackingURL", job.getTrackingURL()); jobReport.putAll(getDetailedJobReport(jobId)); return jobReport; }
From source file:com.twitter.ambrose.model.hadoop.MapReduceHelper.java
License:Apache License
/** * Get the configurations at the beginning of the job flow, it will contain information * about the map/reduce plan and decoded pig script. * @param runningJob/*from w w w . j a v a 2 s. c om*/ * @return Properties - configuration properties of the job */ public void setJobConfFromFile(MapReduceJob job, JobClient jobClient) { Properties jobConfProperties = new Properties(); try { RunningJob runningJob = jobClient.getJob(JobID.forName(job.getId())); if (runningJob == null) { log.warn("Couldn't find job status for jobId: " + job.getId()); } log.info("RunningJob Configuration File location: " + runningJob.getJobFile()); Path path = new Path(runningJob.getJobFile()); Configuration conf = new Configuration(false); FileSystem fileSystem = FileSystem.get(new Configuration()); InputStream inputStream = fileSystem.open(path); conf.addResource(inputStream); Iterator<Map.Entry<String, String>> iter = conf.iterator(); while (iter.hasNext()) { Map.Entry<String, String> entry = iter.next(); jobConfProperties.put(entry.getKey(), entry.getValue()); } } catch (FileNotFoundException e) { log.warn("Configuration file not found for old jobsflows."); } catch (Exception e) { log.warn("Error occurred when retrieving configuration info." + e.getMessage()); } job.setConfiguration(jobConfProperties); }
From source file:edu.stolaf.cs.wmrserver.HadoopEngine.java
License:Apache License
private JobConf loadJobConfiguration(RunningJob job) throws InternalException { // Try normal job file try {/* w w w . jav a 2 s . c o m*/ JobConf conf = new JobConf(); Path jobFile = new Path(job.getJobFile()); FileSystem fs = jobFile.getFileSystem(new Configuration()); conf.addResource(fs.open(jobFile)); return conf; } catch (IOException ex) { } catch (IllegalArgumentException ex) { } // Hadoop 0.20 only return new JobConf(org.apache.hadoop.mapred.JobTracker.getLocalJobFilePath(job.getID())); /* // Try to retrieve configuration from history // Hadoop 0.21 only! try { Method m = JobTracker.class.getMethod("getLocalJobFilePath", JobID.class); String jobFile = m.invoke(null, job.getID()); return new JobConf(jobFile); } catch (NoSuchMethodException ex) { } catch (SecurityException ex) { } // Try to retrieve configuration from history (0.21 only) try { Method getHistoryUrl = job.getClass().getMethod("getHistoryUrl"); Path historyPath = new Path(getHistoryUrl.invoke(job)); Path historyDir = historyPath.getParent(); Class jobHistoryClass = Class.forName( "org.apache.hadoop.mapreduce.jobhistory.JobHistory"); Method getConfFile = jobHistoryClass.getMethod( "getConfFile", Path.class, JobID.class); Path jobFile = getConfFile.invoke(null, historyDir, job.getID()); return new JobConf(jobFile); } catch (IOException ex) { } catch (IllegalArgumentException ex) { // Thrown for empty string in Path // This should only be temporary } return null; */ }
From source file:org.apache.accumulo.server.master.CoordinateRecoveryTask.java
License:Apache License
void cleanupOldJobs() { try {/*from ww w .j av a 2 s .c o m*/ Configuration conf = CachedConfiguration.getInstance(); @SuppressWarnings("deprecation") JobClient jc = new JobClient(new org.apache.hadoop.mapred.JobConf(conf)); for (JobStatus status : jc.getAllJobs()) { if (!status.isJobComplete()) { RunningJob job = jc.getJob(status.getJobID()); if (job.getJobName().equals(LogSort.getJobName())) { log.info("found a running " + job.getJobName()); Configuration jobConfig = new Configuration(false); log.info("fetching configuration from " + job.getJobFile()); jobConfig.addResource(TraceFileSystem .wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration())) .open(new Path(job.getJobFile()))); if (HdfsZooInstance.getInstance().getInstanceID() .equals(jobConfig.get(LogSort.INSTANCE_ID_PROPERTY))) { log.info("Killing job " + job.getID().toString()); } } } } FileStatus[] children = fs.listStatus(new Path(ServerConstants.getRecoveryDir())); if (children != null) { for (FileStatus child : children) { log.info("Deleting recovery directory " + child); fs.delete(child.getPath(), true); } } } catch (IOException e) { log.error("Error cleaning up old Log Sort jobs" + e); } catch (Exception e) { log.error("Unknown error cleaning up old jobs", e); } }
From source file:org.apache.oozie.action.hadoop.TestJavaActionExecutor.java
License:Apache License
public void testACLModifyJob() throws Exception { // CASE 1: If user has provided modify-acl value // then it should NOT be overridden by group name String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node> <configuration>" + "<property><name>mapreduce.job.acl-modify-job</name><value>MODIFIER</value></property>" + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"; Context context = createContext(actionXml, "USERS"); RunningJob job = submitAction(context); FileSystem fs = context.getAppFileSystem(); Configuration jobXmlConf = new XConfiguration(fs.open(new Path(job.getJobFile()))); String userModifyAcl = jobXmlConf.get(JavaActionExecutor.ACL_MODIFY_JOB); // 'MODIFIER' String userGroup = context.getWorkflow().getAcl(); // 'USERS' assertFalse(userGroup.equals(userModifyAcl)); // CASE 2: If user has not provided modify-acl value // then it equals group name actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node> <configuration>" + "</configuration>" + "<main-class>MAIN-CLASS</main-class>" + "</java>"; context = createContext(actionXml, "USERS"); job = submitAction(context);//w w w. j a va2 s. co m fs = context.getAppFileSystem(); jobXmlConf = new XConfiguration(fs.open(new Path(job.getJobFile()))); userModifyAcl = jobXmlConf.get(JavaActionExecutor.ACL_MODIFY_JOB); userGroup = context.getWorkflow().getAcl(); assertTrue(userGroup.equals(userModifyAcl)); }
From source file:org.apache.oozie.action.hadoop.TestOozieJobInfo.java
License:Apache License
public void testInfoWithBundle() throws Exception { Services.get().getConf().setBoolean(OozieJobInfo.CONF_JOB_INFO, true); OozieJobInfo.setJobInfo(true);//from w w w . j av a 2 s .co m BundleJobBean job = this.addRecordToBundleJobTable(Job.Status.PREP, false); final JPAService jpaService = Services.get().get(JPAService.class); Configuration jobConf = null; try { jobConf = new XConfiguration(new StringReader(job.getConf())); } catch (IOException ioe) { log.warn("Configuration parse error. read from DB :" + job.getConf(), ioe); throw new CommandException(ErrorCode.E1005, ioe); } setCoordConf(jobConf); Path appPath = new Path(jobConf.get(OozieClient.BUNDLE_APP_PATH), "bundle.xml"); jobConf.set(OozieClient.BUNDLE_APP_PATH, appPath.toString()); BundleSubmitXCommand submitCmd = new BundleSubmitXCommand(jobConf); submitCmd.call(); BundleJobGetJPAExecutor bundleJobGetExecutor = new BundleJobGetJPAExecutor(submitCmd.getJob().getId()); job = jpaService.execute(bundleJobGetExecutor); assertEquals(job.getStatus(), Job.Status.PREP); new BundleStartXCommand(job.getId()).call(); sleep(2000); List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance() .getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, job.getId()); assertEquals(1, actions.size()); final String bundleID = job.getId(); waitFor(200000, new Predicate() { public boolean evaluate() throws Exception { List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance() .getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, bundleID); return actions.get(0).getStatus().equals(Job.Status.RUNNING); } }); actions = BundleActionQueryExecutor.getInstance() .getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, job.getId()); final String cordID = actions.get(0).getCoordId(); waitFor(200000, new Predicate() { public boolean evaluate() throws Exception { CoordJobGetJPAExecutor coordGetCmd2 = new CoordJobGetJPAExecutor(cordID); CoordinatorJobBean cc = jpaService.execute(coordGetCmd2); return cc.getStatus().equals(Job.Status.RUNNING); } }); final String jobID = jpaService.execute(new WorkflowJobsGetFromCoordParentIdJPAExecutor(cordID, 1)).get(0); final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor = new WorkflowActionsGetForJobJPAExecutor( jobID); waitFor(200000, new Predicate() { public boolean evaluate() throws Exception { List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor); WorkflowActionBean action = null; for (WorkflowActionBean bean : actions) { if (bean.getName().contains("hadoop")) { action = bean; break; } } return action.getStatus().toString().equalsIgnoreCase(Job.Status.RUNNING.toString()); } }); final WorkflowJobGetJPAExecutor wfeExc = new WorkflowJobGetJPAExecutor(jobID); WorkflowJobBean wfbean = jpaService.execute(wfeExc); List<WorkflowActionBean> actionList = jpaService.execute(actionsGetExecutor); ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfbean, actionList.get(1), false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(actionList.get(1).getConf())); String user = conf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf); String launcherId = actionList.get(1).getExternalId(); final RunningJob launcherJob = jobClient.getJob(JobID.forName(launcherId)); FileSystem fs = context.getAppFileSystem(); Configuration jobXmlConf = new XConfiguration(fs.open(new Path(launcherJob.getJobFile()))); String jobInfo = jobXmlConf.get(OozieJobInfo.JOB_INFO_KEY); // BUNDLE_ID;BUNDLE_NAME;COORDINATOR_NAME;COORDINATOR_NOMINAL_TIME; // WORKFLOW_ID;WORKFLOW_NAME;WORKFLOW_DEPTH;WORKFLOW_SUPERPARENT; // ACTION_TYPE;ACTION_NAME,JOB_INFO,custom_info; assertEquals(jobInfo.split(OozieJobInfo.SEPARATOR).length, 13); assertTrue(jobInfo.contains(bundleID)); assertTrue(jobInfo.contains("bundle.name=test_bundle,")); assertTrue(jobInfo.contains(cordID)); assertTrue(jobInfo.contains("action.type=map-reduce")); assertTrue(jobInfo.contains("wf.depth=0")); assertTrue(jobInfo.contains("wf.superparent.id=" + cordID)); assertTrue(jobInfo.contains(",testing=test,")); assertTrue(jobInfo.contains(",coord.nominal.time=")); assertTrue(jobInfo.contains("launcher=true")); }
From source file:org.godhuli.rhipe.FileUtils.java
License:Apache License
public REXP getstatus(String jd, boolean geterrors) throws Exception { org.apache.hadoop.mapred.JobID jj = org.apache.hadoop.mapred.JobID.forName(jd); if (jj == null) throw new IOException("Jobtracker could not find jobID: " + jd); org.apache.hadoop.mapred.RunningJob rj = jclient.getJob(jj); if (rj == null) throw new IOException( "No such job: " + jd + " available, wrong job? or try the History Viewer (see the Web UI) "); String jobfile = rj.getJobFile(); String jobname = rj.getJobName(); // cfg.addResource(new Path(jobfile)); org.apache.hadoop.mapred.Counters cc = rj.getCounters(); long startsec = getStart(jclient, jj); double dura = ((double) System.currentTimeMillis() - startsec) / 1000; REXP ro = FileUtils.buildlistFromOldCounter(cc, dura); int jobs = rj.getJobState(); String jobss = null;//from www. j a v a 2 s. c o m if (jobs == JobStatus.FAILED) jobss = "FAILED"; else if (jobs == JobStatus.KILLED) jobss = "KILLED"; else if (jobs == JobStatus.PREP) jobss = "PREP"; else if (jobs == JobStatus.RUNNING) jobss = "RUNNING"; else if (jobs == JobStatus.SUCCEEDED) jobss = "SUCCEEDED"; float mapprog = rj.mapProgress(), reduprog = rj.reduceProgress(); org.apache.hadoop.mapred.TaskReport[] maptr = jclient.getMapTaskReports(jj); org.apache.hadoop.mapred.TaskReport[] redtr = jclient.getReduceTaskReports(jj); int totalmaps = maptr.length, totalreds = redtr.length; int mappending = 0, redpending = 0, maprunning = 0, redrunning = 0, redfailed = 0, redkilled = 0, mapkilled = 0, mapfailed = 0, mapcomp = 0, redcomp = 0; for (int i = 0; i < maptr.length; i++) { TIPStatus t = maptr[i].getCurrentStatus(); switch (t) { case COMPLETE: mapcomp++; break; case FAILED: mapfailed++; break; case PENDING: mappending++; break; case RUNNING: maprunning++; break; case KILLED: mapkilled++; break; } } for (int i = 0; i < redtr.length; i++) { TIPStatus t = redtr[i].getCurrentStatus(); switch (t) { case COMPLETE: redcomp++; break; case FAILED: redfailed++; break; case PENDING: redpending++; break; case RUNNING: redrunning++; break; case KILLED: redkilled++; break; } } int reduceafails = 0, reduceakilled = 0, mapafails = 0, mapakilled = 0; int startfrom = 0; REXP.Builder errcontainer = REXP.newBuilder(); errcontainer.setRclass(REXP.RClass.STRING); while (true) { org.apache.hadoop.mapred.TaskCompletionEvent[] events = rj.getTaskCompletionEvents(startfrom); for (int i = 0; i < events.length; i++) { org.apache.hadoop.mapred.TaskCompletionEvent e = events[i]; int f = 0, k = 0; switch (e.getTaskStatus()) { case KILLED: if (e.isMapTask()) { mapakilled++; } else { reduceakilled++; } break; case TIPFAILED: case FAILED: if (e.isMapTask()) { mapafails++; } else { reduceafails++; } if (geterrors) { REXPProtos.STRING.Builder content = REXPProtos.STRING.newBuilder(); String[] s = rj.getTaskDiagnostics(e.getTaskAttemptId()); if (s != null && s.length > 0) { content.setStrval(s[0]); errcontainer.addStringValue(content.build()); } } break; } } startfrom += events.length; if (events.length == 0) break; } REXP.Builder thevals = REXP.newBuilder(); thevals.setRclass(REXP.RClass.LIST); thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobss })); thevals.addRexpValue(RObjects.buildDoubleVector(new double[] { dura })); thevals.addRexpValue(RObjects.buildDoubleVector(new double[] { (double) mapprog, (double) reduprog })); thevals.addRexpValue(RObjects.buildIntVector( new int[] { totalmaps, mappending, maprunning, mapcomp, mapkilled, mapafails, mapakilled })); thevals.addRexpValue(RObjects.buildIntVector( new int[] { totalreds, redpending, redrunning, redcomp, redkilled, reduceafails, reduceakilled })); thevals.addRexpValue(ro); thevals.addRexpValue(errcontainer); thevals.addRexpValue(RObjects.makeStringVector(rj.getTrackingURL())); thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobname })); thevals.addRexpValue(RObjects.makeStringVector(new String[] { jobfile })); return (thevals.build()); }
From source file:org.godhuli.rhipe.FileUtils.java
License:Apache License
public REXP joinjob(String jd, boolean verbose) throws Exception { org.apache.hadoop.mapred.JobID jj = org.apache.hadoop.mapred.JobID.forName(jd); org.apache.hadoop.mapred.RunningJob rj = jclient.getJob(jj); String jobfile = rj.getJobFile(); // cfg.addResource(new Path(jobfile)); boolean issuc = jclient.monitorAndPrintJob(jobconf, rj); org.apache.hadoop.mapred.Counters cc = rj.getCounters(); long startsec = getStart(jclient, jj); REXP ro = FileUtils.buildlistFromOldCounter(cc, ((double) System.currentTimeMillis() - startsec) / 1000); REXP.Builder thevals = REXP.newBuilder(); thevals.setRclass(REXP.RClass.LIST); thevals.addRexpValue(ro);/*ww w. jav a 2 s . c o m*/ thevals.addRexpValue(RObjects.buildBooleanVector(new boolean[] { issuc }).build()); return (thevals.build()); }