List of usage examples for org.apache.hadoop.mapred RunningJob getJobState
public int getJobState() throws IOException;
From source file:com.jackbe.mapreduce.LocalJobManager.java
License:Open Source License
@Override public String getJobState(RunningJob job) throws IOException { if (log.isTraceEnabled()) log.trace("called."); if (job == null) return JOB_NOT_FOUND; return JobStatus.getJobRunState(job.getJobState()); }
From source file:com.jackbe.mapreduce.LocalJobManager.java
License:Open Source License
@Override public String getAllJobs() { RunningJob[] jobsInfo = null;// w w w. jav a 2 s .c o m try { if (jobClient == null) return "<Exception> NULL jobClient. </Exception>"; //FIXME need to fix for remote jobs //jobsInfo = jobClient.getAllJobs(); } catch (Exception e) { log.error("Exception getting jobs from jobClient: " + e, e); return "Exception getting jobs: " + e.getMessage(); } // If this is running local, we need to use the Jobs in our map. // TODO: remove old entries eventually. RunningJob[] temp = new RunningJob[statusMap.size()]; jobsInfo = statusMap.values().toArray(temp); StringBuffer xml = new StringBuffer(); xml.append("<jobs>\n"); for (RunningJob job : jobsInfo) { try { xml.append("<job>\n\t<id>" + job.getID().toString() + "</id>\n\t<state>" + JobStatus.getJobRunState(job.getJobState())); } catch (IOException e) { log.error("Exception apending job status info: " + e); // The XML string is now screwed up, just return null. return null; } xml.append("</state>\n\t<progress>" + getJobProgress(job.getID().toString()) + "</progress>\n</job>\n"); } xml.append("\n</jobs>"); return xml.toString(); }
From source file:com.jackbe.mapreduce.LocalJobManager.java
License:Open Source License
@Override public void clearCompletedJobs() { if (log.isTraceEnabled()) log.trace("called."); for (String key : statusMap.keySet()) { RunningJob job = statusMap.get(key); try {/*w w w .ja v a 2s .c om*/ if (job.getJobState() != JobStatus.RUNNING) statusMap.remove(key); } catch (IOException e) { log.error("Exception clearing completed jobs: " + e, e); } } }
From source file:com.ngdata.hbaseindexer.master.BatchStateUpdater.java
License:Apache License
@Override public void run() { IndexerDefinition indexerDefinition = null; try {//from w w w .j a va 2s . co m indexerDefinition = indexerModel.getIndexer(indexerName); } catch (IndexerNotFoundException e) { log.info("Could not find index " + indexerName + " while checking batch rebuild status.", e); } if (indexerDefinition != null) { log.debug("Checking batch state for " + indexerDefinition.getName()); BatchBuildInfo batchBuildInfo = indexerDefinition.getActiveBatchBuildInfo(); if (batchBuildInfo != null) { Set<String> jobs = batchBuildInfo.getMapReduceJobTrackingUrls().keySet(); boolean batchDone = true; boolean overAllSuccess = true; for (String jobId : jobs) { RunningJob job; try { job = jobClient.getJob(JobID.forName(jobId)); } catch (IOException e) { log.error("Could not get job " + jobId + " for index " + indexerDefinition.getName() + " while checking active build info.", e); batchDone = false; break; } if (job != null) { int jobState; try { jobState = job.getJobState(); } catch (IOException e) { log.error( "Could not get jobstate for job " + jobId + " for index " + indexerDefinition.getName() + " while checking active build info.", e); batchDone = false; break; } batchDone = batchDone && jobState != JobStatus.RUNNING; overAllSuccess = overAllSuccess && jobState == JobStatus.SUCCEEDED; } else { log.warn("Could not find job " + jobId + " while checking active batch builds for indexer " + indexerDefinition.getName()); } } if (batchDone) { markBatchBuildCompleted(indexerDefinition.getName(), overAllSuccess); } else { executor.schedule(this, pollInterval, TimeUnit.MILLISECONDS); } } } }
From source file:com.ngdata.hbaseindexer.master.BatchStateUpdaterTest.java
License:Apache License
@Test public void testRun_Running() throws Exception { String jobId = "job_201407251005_0815"; createDefinition("mytest", jobId); RunningJob job = createJob(jobId, JobStatus.RUNNING); when(job.getJobState()).thenReturn(JobStatus.RUNNING); Assert.assertEquals(0, executorService.getQueue().size()); checkAllIndexes();/* w w w . j a va 2 s. co m*/ Assert.assertEquals(1, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(1)).getIndexer(anyString()); verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class)); Thread.sleep(60); Assert.assertEquals(1, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(2)).getIndexer(anyString()); verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class)); when(job.getJobState()).thenReturn(JobStatus.SUCCEEDED); Thread.sleep(60); Assert.assertEquals(0, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(3)).getIndexer(anyString()); verify(model, VerificationModeFactory.times(1)).updateIndexerInternal(any(IndexerDefinition.class)); }
From source file:com.ngdata.hbaseindexer.master.BatchStateUpdaterTest.java
License:Apache License
@Test public void testRun_MultiIndex() throws Exception { String jobId1 = "job_201407251005_0001"; String jobId2 = "job_201407251005_0002"; createDefinition("mytest1", jobId1); RunningJob job1 = createJob(jobId1, JobStatus.RUNNING); createDefinition("mytest2", jobId2); RunningJob job2 = createJob(jobId2, JobStatus.RUNNING); Assert.assertEquals(0, executorService.getQueue().size()); checkAllIndexes();/*from w w w . j a va 2 s . c o m*/ Assert.assertEquals(2, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class)); Thread.sleep(60); Assert.assertEquals(2, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(0)).updateIndexerInternal(any(IndexerDefinition.class)); when(job1.getJobState()).thenReturn(JobStatus.SUCCEEDED); Thread.sleep(60); Assert.assertEquals(1, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(1)).updateIndexerInternal(any(IndexerDefinition.class)); when(job2.getJobState()).thenReturn(JobStatus.SUCCEEDED); Thread.sleep(60); Assert.assertEquals(0, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(2)).updateIndexerInternal(any(IndexerDefinition.class)); Thread.sleep(60); Assert.assertEquals(0, executorService.getQueue().size()); verify(model, VerificationModeFactory.times(2)).updateIndexerInternal(any(IndexerDefinition.class)); }
From source file:com.ngdata.hbaseindexer.master.BatchStateUpdaterTest.java
License:Apache License
private RunningJob createJob(String jobId, int status) throws IOException { RunningJob job = mock(RunningJob.class); when(job.getJobState()).thenReturn(status); when(jobClient.getJob(JobID.forName(jobId))).thenReturn(job); return job;//from ww w . ja v a 2s . c o m }
From source file:de.tudarmstadt.ukp.dkpro.bigdata.hadoop.DkproHadoopDriver.java
License:Apache License
/** * Runs the UIMA pipeline./*from w w w. ja v a 2 s . c o m*/ * * @return 0 if Hadoop job succeeded, 1 if job failed, 2 if it was killed, otherwise 3 * * @see org.apache.hadoop.util.Tool#run(java.lang.String[]) */ @Override public int run(String[] args) throws Exception { if (args.length < 2) { System.out.println( "Usage: " + this.getClass().getSimpleName() + " [hadoop-params] input output [job-params]"); System.exit(1); } this.job = new JobConf(getConf(), DkproHadoopDriver.class); final FileSystem fs = FileSystem.get(this.job); // set the factory class name this.job.set("dkpro.uima.factory", this.getClass().getName()); Path inputPath; if (args[0].contains(",")) { String[] inputPaths = args[0].split(","); inputPath = new Path(inputPaths[0]); for (String path : inputPaths) FileInputFormat.addInputPath(job, new Path(path)); } else { inputPath = new Path(args[0]); // input FileInputFormat.setInputPaths(this.job, inputPath); } final Path outputPath = new Path(args[1]);// output final CollectionReader reader = buildCollectionReader(); // if a collection reader was defined, import data into hdfs // try { // final Class<?> c = Class.forName("org.apache.hadoop.io.compress.SnappyCodec"); // FileOutputFormat.setOutputCompressorClass(this.job, // (Class<? extends CompressionCodec>) c); // } // catch (final Exception e) { // // } if (reader != null) { final AnalysisEngine xcasWriter = AnalysisEngineFactory.createEngine( CASWritableSequenceFileWriter.class, // createTypeSystemDescription(), CASWritableSequenceFileWriter.PARAM_PATH, inputPath.toString(), CASWritableSequenceFileWriter.PARAM_COMPRESS, true, CASWritableSequenceFileWriter.PARAM_FS, job.get(("fs.default.name"), "file:/")); runPipeline(reader, xcasWriter); } // cleanup previous output fs.delete(outputPath, true); // this is a sensible default for the UKP cluster int numMappers = 256; // if (args.length > 2) { // numMappers = Integer.parseInt(args[2]); // } FileOutputFormat.setOutputPath(this.job, outputPath); // SequenceFileOutputFormat.setCompressOutput(this.job, true); if (this.job.get("mapred.output.compress") == null) { this.job.setBoolean("mapred.output.compress", true); } // Just in case compression is on this.job.set("mapred.output.compression.type", "BLOCK"); if (this.job.getBoolean("dkpro.output.plaintext", false)) { this.job.setOutputFormat(TextOutputFormat.class); } else { this.job.setOutputFormat(SequenceFileOutputFormat.class); } // this.job.set("mapred.output.compression.codec", // "org.apache.hadoop.io.compress.GzipCodec"); // use compression // setup some sensible defaults this.job.setMapperClass(this.mapperClass); this.job.setReducerClass(this.reducerClass); if (getInputFormatClass() != null) { this.job.setInputFormat(getInputFormatClass()); } else { this.job.setInputFormat(SequenceFileInputFormat.class); } // this.job.setOutputFormat(TextOutputFormat.class); this.job.setMapOutputKeyClass(Text.class); this.job.setMapOutputValueClass(BinCasWithTypeSystemWritable.class); this.job.setOutputKeyClass(Text.class); this.job.setOutputValueClass(BinCasWithTypeSystemWritable.class); this.job.setJobName(this.getClass().getSimpleName()); // this.job.set("mapred.child.java.opts", "-Xmx1g"); this.job.setInt("mapred.job.map.memory.mb", 1280); this.job.setInt("mapred.job.reduce.memory.mb", 1280); this.job.setNumMapTasks(numMappers); this.job.setNumReduceTasks(0); configure(this.job); // create symlinks for distributed resources DistributedCache.createSymlink(this.job); // sLogger.info("Running job "+job.getJobName()); RunningJob runningJob = JobClient.runJob(this.job); runningJob.waitForCompletion(); int status = runningJob.getJobState(); if (status == JobStatus.SUCCEEDED) { return 0; } else if (status == JobStatus.FAILED) { return 1; } else if (status == JobStatus.KILLED) { return 2; } else { return 3; } }
From source file:edu.stolaf.cs.wmrserver.HadoopEngine.java
License:Apache License
public JobStatus getStatus(Submission submission) throws NotFoundException, InternalException { RunningJob job = getJob(submission); JobConf conf = loadJobConfiguration(job); JobStatus status = new JobStatus(); status.setInfo(getInfo(submission, job, conf)); try {//from w w w.j a va2 s .co m JobClient client = new JobClient(new JobConf()); // Get job state // Thanks to the mentally handicapped switch statement, we have // to use a chain of ifs. Fuck Java. int jobState = job.getJobState(); if (jobState == org.apache.hadoop.mapred.JobStatus.FAILED) status.setState(State.FAILED); else if (jobState == org.apache.hadoop.mapred.JobStatus.SUCCEEDED) status.setState(State.SUCCESSFUL); else if (jobState == org.apache.hadoop.mapred.JobStatus.KILLED) status.setState(State.KILLED); else if (jobState == org.apache.hadoop.mapred.JobStatus.RUNNING) status.setState(State.RUNNING); else status.setState(State.PREP); // Get task counts TaskReport[] mapTaskReports = client.getMapTaskReports(job.getID()); TaskReport[] reduceTaskReports = client.getReduceTaskReports(job.getID()); // Get failed task logs TaskCompletionEvent[] events = job.getTaskCompletionEvents(0); Pair<ArrayList<TaskLog>, ArrayList<TaskLog>> failures; if (events != null) failures = getLogsFromCompletionEvents(events); else failures = getLogsFromHistory(job, new Configuration()); ArrayList<TaskLog> mapFailures = failures.first; ArrayList<TaskLog> reduceFailures = failures.second; // Get other mapper info PhaseStatus mapStatus = new PhaseStatus(); mapStatus.setProgress(job.mapProgress() * 100); if (!mapFailures.isEmpty()) mapStatus.setErrors(getMeaningfulTaskLog(mapFailures)); if (mapTaskReports != null) mapStatus.setTotalTasks(mapTaskReports.length); // TODO: Handle the state in a sane way mapStatus.setState(status.getState()); status.setMapStatus(mapStatus); // Get other reducer info PhaseStatus reduceStatus = new PhaseStatus(); reduceStatus.setProgress(job.reduceProgress() * 100); if (!reduceFailures.isEmpty()) reduceStatus.setErrors(getMeaningfulTaskLog(reduceFailures)); reduceStatus.setState(status.getState()); if (reduceTaskReports != null) reduceStatus.setTotalTasks(reduceTaskReports.length); if (conf != null) reduceStatus.setOutputPath(FileOutputFormat.getOutputPath(conf).toString()); status.setReduceStatus(reduceStatus); } catch (Exception ex) { throw JobServiceHandler.wrapException("Could not get job info.", ex); } return status; }
From source file:FormatStorage1.MergeFileUtil.java
License:Open Source License
public static void run(String inputdir, String outputdir, Configuration conf) throws IOException { JobConf job = new JobConf(conf); job.setJobName("MergeFileUtil"); job.setJarByClass(MergeFileUtil.class); FileSystem fs = null;/*from w ww. j a v a 2 s. c om*/ fs = FileSystem.get(job); if (fs.exists(new Path(outputdir))) { throw new IOException("outputdir: " + outputdir + " exist!!!"); } FileStatus[] fss = fs.listStatus(new Path(inputdir)); if (fss == null || fss.length <= 0) { throw new IOException("no input files"); } IFormatDataFile ifdf = new IFormatDataFile(job); ifdf.open(fss[0].getPath().toString()); job.set("ifdf.head.info", ifdf.fileInfo().head().toStr()); ifdf.close(); long wholesize = 0; for (FileStatus status : fss) { wholesize += status.getLen(); } job.setNumReduceTasks(0); FileInputFormat.setInputPaths(job, inputdir); FileOutputFormat.setOutputPath(job, new Path(outputdir)); job.setOutputKeyClass(LongWritable.class); job.setOutputValueClass(IRecord.class); job.setMapperClass(MergeMap.class); job.setInputFormat(CombineFormatStorageFileInputFormat.class); job.setOutputFormat(MergeIFormatOutputFormat.class); JobClient jc = new JobClient(job); RunningJob rjob = jc.submitJob(job); try { String lastReport = ""; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss,SSS"); long reportTime = System.currentTimeMillis(); long maxReportInterval = 3 * 1000; while (!rjob.isComplete()) { Thread.sleep(1000); int mapProgress = Math.round(rjob.mapProgress() * 100); int reduceProgress = Math.round(rjob.reduceProgress() * 100); String report = " map = " + mapProgress + "%, reduce = " + reduceProgress + "%"; if (!report.equals(lastReport) || System.currentTimeMillis() >= reportTime + maxReportInterval) { String output = dateFormat.format(Calendar.getInstance().getTime()) + report; System.err.println(output); lastReport = report; reportTime = System.currentTimeMillis(); } } LOG.info(rjob.getJobState()); } catch (IOException e1) { e1.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } }