List of usage examples for org.apache.hadoop.mapred JobClient JobClient
public JobClient(InetSocketAddress jobTrackAddr, Configuration conf) throws IOException
From source file:com.google.HadoopMonitor.java
License:Open Source License
public HadoopMonitor() throws IOException { jobClient = new JobClient(new InetSocketAddress("hadoop-jobtracker", 9001), new Configuration()); gson = new Gson(); jobs = new HashMap<String, JobState>(); client = new HttpsClient(); }
From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java
License:Open Source License
public HadoopJobRunner() { bLocalMode = prop_custom.getHadoopLocalMode(); try {/* ww w . j a v a 2s.co m*/ @SuppressWarnings("unused") JobClient jc = new JobClient(getJobClientConnection(), new Configuration()); if (bLocalMode) { System.out.println("Will run hadoop locally (infrastructure appears to exist)."); } } catch (Exception e) { // Hadoop doesn't work if (bLocalMode) { System.out.println("Will run hadoop locally (no infrastructure)."); } else { System.out.println("No hadoop infrastructure installed, will just look for saved queries."); } bHadoopEnabled = false; } }
From source file:com.ikanow.infinit.e.core.mapreduce.HadoopJobRunner.java
License:Open Source License
/** * Checks any running/queued jobs and updates their status if they've completed *//*w w w .j a v a2 s .c o m*/ public void updateJobStatus() { Map<ObjectId, String> incompleteJobsMap = new HashMap<ObjectId, String>(); //get mongo entries that have jobids? try { JobClient jc = null; CustomMapReduceJobPojo cmr = getJobsToMakeComplete(); while (cmr != null) { boolean markedComplete = false; //make sure its an actual ID, we now set jobidS to "" when running the job if (!cmr.jobidS.equals("")) { if (null == jc) { try { jc = new JobClient(getJobClientConnection(), new Configuration()); } catch (Exception e) { // Better delete this, no idea what's going on.... _logger.info( "job_update_status_error_title=" + cmr.jobtitle + " job_update_status_error_id=" + cmr._id.toString() + " job_update_status_error_message=Skipping job: " + cmr.jobidS + cmr.jobidN + ", this node does not run mapreduce"); setJobComplete(cmr, true, true, -1, -1, "Failed to launch job, unknown error (check configuration in /opt/hadoop-infinite/mapreduce/hadoop/, jobtracker may be localhost?)."); cmr = getJobsToMakeComplete(); continue; } } //check if job is done, and update if it is JobStatus[] jobs = jc.getAllJobs(); boolean bFound = false; for (JobStatus j : jobs) { if (j.getJobID().getJtIdentifier().equals(cmr.jobidS) && j.getJobID().getId() == cmr.jobidN) { bFound = true; boolean error = false; markedComplete = j.isJobComplete(); String errorMessage = null; if (JobStatus.FAILED == j.getRunState()) { markedComplete = true; error = true; errorMessage = "Job failed while running, check for errors in the mapper/reducer or that your key/value classes are set up correctly?"; } setJobComplete(cmr, markedComplete, error, j.mapProgress(), j.reduceProgress(), errorMessage); break; // (from mini loop over hadoop jobs, not main loop over infinite tasks) } } if (!bFound) { // Possible error //check if its been longer than 5min and mark job as complete (it failed to launch) Date currDate = new Date(); Date lastDate = cmr.lastRunTime; //if its been more than 5 min (5m*60s*1000ms) if (currDate.getTime() - lastDate.getTime() > 300000) { markedComplete = true; setJobComplete(cmr, true, true, -1, -1, "Failed to launch job, unknown error #2."); } } } else // this job hasn't been started yet: { //check if its been longer than 5min and mark job as complete (it failed to launch) Date currDate = new Date(); Date lastDate = cmr.lastRunTime; //if its been more than 5 min (5m*60s*1000ms) if (currDate.getTime() - lastDate.getTime() > 300000) { markedComplete = true; setJobComplete(cmr, true, true, -1, -1, "Failed to launch job, unknown error #1."); } } //job was not done, need to set flag back if (!markedComplete) { incompleteJobsMap.put(cmr._id, cmr.jobidS); } cmr = getJobsToMakeComplete(); } } catch (Exception ex) { _logger.info("job_error_checking_status_message=" + HarvestExceptionUtils.createExceptionMessage(ex)); } catch (Error err) { // Really really want to get to the next line of code, and clear the status... } //set all incomplete jobs back for (ObjectId id : incompleteJobsMap.keySet()) { BasicDBObject update = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, incompleteJobsMap.get(id)); DbManager.getCustom().getLookup().update(new BasicDBObject(CustomMapReduceJobPojo._id_, id), new BasicDBObject(MongoDbManager.set_, update)); } }
From source file:com.ikanow.infinit.e.processing.custom.CustomProcessingController.java
License:Open Source License
public CustomProcessingController(Integer debugLimit) { prop_custom = new com.ikanow.infinit.e.processing.custom.utils.PropertiesManager(); _statusManager = new CustomStatusManager(prop_custom); _nDebugLimit = debugLimit;/* w w w .j a v a2 s .co m*/ _bLocalMode = prop_custom.getHadoopLocalMode(); try { @SuppressWarnings("unused") JobClient jc = new JobClient(InfiniteHadoopUtils.getJobClientConnection(prop_custom), new Configuration()); if (_bLocalMode) { System.out.println("Will run hadoop locally (infrastructure appears to exist)."); } } catch (Error e) { if (_bLocalMode) { System.out.println("Will run hadoop locally (no infrastructure)."); } else { System.out.println("No hadoop infrastructure installed, will just look for saved queries."); } _bHadoopEnabled = false; } catch (Exception e) { // Hadoop doesn't work if (_bLocalMode) { System.out.println("Will run hadoop locally (no infrastructure)."); } else { System.out.println("No hadoop infrastructure installed, will just look for saved queries."); } _bHadoopEnabled = false; } }
From source file:com.ikanow.infinit.e.processing.custom.CustomProcessingController.java
License:Open Source License
public boolean killRunningJob(CustomMapReduceJobPojo jobToKillInfo) { try {/* w w w .j a va 2 s. c o m*/ Configuration conf = new Configuration(); JobClient jc = new JobClient(InfiniteHadoopUtils.getJobClientConnection(prop_custom), conf); jc.setConf(conf); // (doesn't seem to be set by the above call) RunningJob jobToKill = jc.getJob(new JobID(jobToKillInfo.jobidS, jobToKillInfo.jobidN)); if (null == jobToKill) { _logger.error("Couldn't find this job: " + jobToKillInfo.jobidS + "_" + jobToKillInfo.jobidN + " / " + new JobID(jobToKillInfo.jobidS, jobToKillInfo.jobidN).toString()); return false; } jobToKill.killJob(); int nRuns = 0; while (!checkRunningJobs(jobToKillInfo)) { try { Thread.sleep(5000); } catch (Exception e) { } if (++nRuns > 24) { // bail out after 2 minutes _logger.error("Killed job: " + jobToKillInfo.jobidS + "_" + jobToKillInfo.jobidN + ", but job failed to stop within time allowed"); return false; } } if (null != jobToKillInfo.derivedFromSourceKey) { // Update the derived source, if one existse BasicDBObject query = new BasicDBObject(SourcePojo.key_, jobToKillInfo.derivedFromSourceKey); BasicDBObject setUpdate = new BasicDBObject(SourceHarvestStatusPojo.sourceQuery_harvest_status_, HarvestEnum.error.toString()); setUpdate.put(SourceHarvestStatusPojo.sourceQuery_harvest_message_, "Manually stopped"); BasicDBObject srcUpdate = new BasicDBObject(DbManager.set_, setUpdate); DbManager.getIngest().getSource().update(query, srcUpdate, false, false); } //TESTED (actually a bit pointless usually because is then overwritten by the source publish) return true; } catch (Exception e) { _logger.error("Failed to kill job: " + jobToKillInfo.jobidS + "_" + jobToKillInfo.jobidN + " / " + e.getMessage(), e); return false; } }
From source file:com.ikanow.infinit.e.processing.custom.CustomProcessingController.java
License:Open Source License
public boolean checkRunningJobs(CustomMapReduceJobPojo jobOverride) { Map<ObjectId, String> incompleteJobsMap = new HashMap<ObjectId, String>(); //get mongo entries that have jobids? try {/* w w w . ja v a2s.c o m*/ JobClient jc = null; CustomMapReduceJobPojo cmr = jobOverride; if (null == cmr) cmr = CustomScheduleManager.getJobsToMakeComplete(_bHadoopEnabled, incompleteJobsMap); else if (null == cmr.jobidS) return true; while (cmr != null) { boolean markedComplete = false; //make sure its an actual ID, we now set jobidS to "" when running the job if (!cmr.jobidS.equals("")) // non null by construction { if (null == jc) { try { jc = new JobClient(InfiniteHadoopUtils.getJobClientConnection(prop_custom), new Configuration()); } catch (Exception e) { // Better delete this, no idea what's going on.... _logger.info( "job_update_status_error_title=" + cmr.jobtitle + " job_update_status_error_id=" + cmr._id.toString() + " job_update_status_error_message=Skipping job: " + cmr.jobidS + cmr.jobidN + ", this node does not run mapreduce"); _statusManager.setJobComplete(cmr, true, true, -1, -1, "Failed to launch job, unknown error (check configuration in /opt/hadoop-infinite/mapreduce/hadoop/, jobtracker may be localhost?)."); incompleteJobsMap.remove(cmr._id); cmr = CustomScheduleManager.getJobsToMakeComplete(_bHadoopEnabled, incompleteJobsMap); continue; } } //check if job is done, and update if it is JobStatus[] jobs = jc.getAllJobs(); boolean bFound = false; for (JobStatus j : jobs) { if (j.getJobID().getJtIdentifier().equals(cmr.jobidS) && j.getJobID().getId() == cmr.jobidN) { bFound = true; boolean error = false; markedComplete = j.isJobComplete(); String errorMessage = null; if (JobStatus.FAILED == j.getRunState()) { markedComplete = true; error = true; errorMessage = "Job failed while running, check for errors in the mapper/reducer or that your key/value classes are set up correctly? " + j.getFailureInfo(); } _statusManager.setJobComplete(cmr, markedComplete, error, j.mapProgress(), j.reduceProgress(), errorMessage); break; // (from mini loop over hadoop jobs, not main loop over infinite tasks) } } if (!bFound) { // Possible error //check if its been longer than 5min and mark job as complete (it failed to launch) Date currDate = new Date(); Date lastDate = cmr.lastRunTime; //if its been more than 5 min (5m*60s*1000ms) if (currDate.getTime() - lastDate.getTime() > 300000) { markedComplete = true; _statusManager.setJobComplete(cmr, true, true, -1, -1, "Failed to launch job, unknown error #2."); } } } else // this job hasn't been started yet: { //check if its been longer than 5min and mark job as complete (it failed to launch) Date currDate = new Date(); Date lastDate = cmr.lastRunTime; //if its been more than 5 min (5m*60s*1000ms) if (currDate.getTime() - lastDate.getTime() > 300000) { markedComplete = true; _statusManager.setJobComplete(cmr, true, true, -1, -1, "Failed to launch job, unknown error #1."); } } //job was done, remove flag if (markedComplete) { incompleteJobsMap.remove(cmr._id); } if (null == jobOverride) cmr = CustomScheduleManager.getJobsToMakeComplete(_bHadoopEnabled, incompleteJobsMap); else cmr = null; } } catch (Exception ex) { _logger.info("job_error_checking_status_message=" + InfiniteHadoopUtils.createExceptionMessage(ex)); } catch (Error err) { // Really really want to get to the next line of code, and clear the status... _logger.info("job_error_checking_status_message=" + InfiniteHadoopUtils.createExceptionMessage(err)); } if (null == jobOverride) { //set all incomplete jobs' status back for (ObjectId id : incompleteJobsMap.keySet()) { BasicDBObject update = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, incompleteJobsMap.get(id)); DbManager.getCustom().getLookup().update(new BasicDBObject(CustomMapReduceJobPojo._id_, id), new BasicDBObject(MongoDbManager.set_, update)); } } return incompleteJobsMap.isEmpty(); }
From source file:com.impetus.ankush2.hadoop.monitor.JobStatusProvider.java
License:Open Source License
/** * Gets the job client.//w ww .j ava 2 s . c om * * @return JobClient object */ private JobClient getJobClient(String host, String port) { // JobClient is the primary interface for the user-job to interact with // the JobTracker. JobClient jobClient = null; if (host == null) { host = "localhost"; } LOG.info("Requesting job Client.."); try { // Provides access to configuration parameters. Configuration conf = new Configuration(); LOG.info("JobClient : " + host + " & port : " + port); // Build a job client, connect to the indicated job tracker. jobClient = new JobClient(new InetSocketAddress(host, Integer.parseInt(port)), new JobConf(conf)); // Set the configuration to be used by this object. jobClient.setConf(conf); } catch (Exception e) { LOG.error(e.getMessage(), e); } return jobClient; }
From source file:com.jackbe.mapreduce.LocalJobManager.java
License:Open Source License
public void init() { conf = new JobConf(LocalJobManager.class); //conf.set("mapred.map.child.java.opts", "-Djava.library.path=/"); //conf.set("mapred.reduce.child.java.opts", "-Djava.library.path=/"); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(Text.class); conf.set("mapred.job.tracker", "ip-10-244-163-80:45365"); // conf.set("mapred.job.tracker","ec2-50-16-77-94.compute-1.amazonaws.com:45365"); conf.setMapperClass(EMMLMapper.class); conf.setReducerClass(EMMLReducer.class); conf.setJar("/home/ec2-user/hadoopmapreduce.jar"); conf.set("mapper.scriptname", this.mapperScriptName); conf.set("reducer.scriptname", this.reducerScriptName); //conf.set("combiner.scriptname", this.combinerScriptName); //conf.setNumTasksToExecutePerJvm(100); //conf.setNumMapTasks(250); //conf.setNumTasksToExecutePerJvm(50); //conf.setNumReduceTasks(50); conf.setInputFormat(TextInputFormat.class); //Use our own XML formatter conf.setOutputFormat(XmlOutputFormat.class); try {//from w ww . jav a 2 s .c om jobClient = new JobClient(JobTracker.getAddress(conf), conf); } catch (IOException e) { log.error("Exception creating jobClient: " + e, e); } }
From source file:eu.scape_project.tb.hadoopjobtracker.HadoobJobTrackerClient.java
License:Apache License
public HadoobJobTrackerClient(String jobTrackerHostName, int jobTrackerHostPort) { logger.info("jobTrackerHostName: " + jobTrackerHostName + "; jobTrackerHostPort: " + jobTrackerHostPort); Configuration conf = new Configuration(); try {// ww w . j a va 2 s. c o m myJobClient = new JobClient(new InetSocketAddress(jobTrackerHostName, jobTrackerHostPort), conf); myJobClient.setConf(conf); } catch (IOException ex) { logger.error("Error connecting to the JobTracker. ERR: " + ex.getMessage()); } }
From source file:org.apache.ambari.servicemonitor.clients.JTListQueue.java
License:Apache License
@Override protected Operation executeOneOperation() throws IOException { Operation operation = new Operation("lsqueue " + queuename); started(operation);//from ww w . ja va2s . c o m JobClient jobClient = null; try { jobClient = new JobClient(jtAddr, getConf()); JobQueueInfo queueInfo = jobClient.getQueueInfo(queuename); operation.setText("Queue " + queuename + " is in state " + queueInfo.getQueueState() + "; scheduling info: " + queueInfo.getSchedulingInfo()); operation.success(); } catch (ExitClientRunException e) { //propagate this up throw e; } catch (IOException e) { //all other outcomes are failures operation.failure(e); } finally { MonitorUtils.closeJobClient(jobClient); } return operation; }