List of usage examples for org.apache.hadoop.mapreduce Job reduceProgress
public float reduceProgress() throws IOException
From source file:cascading.stats.hadoop.HadoopStepStats.java
License:Open Source License
/** * Returns the underlying Reduce tasks progress percentage. * <p/>//from w w w .j a va 2s.c o m * This method is experimental. * * @return float */ public float getReduceProgress() { Job runningJob = getJob(getJobStatusClient()); if (runningJob == null) return 0; try { return runningJob.reduceProgress(); } catch (IOException exception) { throw new FlowException("unable to get progress"); } }
From source file:com.cloudera.oryx.computation.common.JobStep.java
License:Open Source License
/** * @return three progress values, in [0,1], as a {@code float[]}, representing setup, mapper and reducer progress *//*ww w . j a v a 2 s. c o m*/ private float[] determineProgresses() throws IOException, InterruptedException { if (exec == null) { return null; } Cluster cluster = new Cluster(getConf()); try { JobID jobID = getJob().getJobID(); if (jobID == null) { return null; } Job runningJob = cluster.getJob(jobID); if (runningJob == null) { return null; } return new float[] { runningJob.setupProgress(), runningJob.mapProgress(), runningJob.reduceProgress() }; } finally { cluster.close(); } }
From source file:org.apache.blur.mapreduce.lib.BlurOutputFormatTest.java
License:Apache License
public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException, ClassNotFoundException { Path input = getInDir();/*from w w w.j ava 2s. com*/ Path output = getOutDir(); _fileSystem.delete(input, true); _fileSystem.delete(output, true); // 1500 * 50 = 75,000 writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1"); // 100 * 5000 = 500,000 writeRecordsFile(new Path(input, "part2"), 1, 5000, 2000, 100, "cf1"); Job job = Job.getInstance(_conf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, input); CsvBlurMapper.addColumns(job, "cf1", "col"); Path tablePath = new Path(new Path(_root, "table"), "test"); TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(2); tableDescriptor.setTableUri(tablePath.toString()); tableDescriptor.setName("test"); createShardDirectories(getOutDir(), 2); BlurOutputFormat.setupJob(job, tableDescriptor); BlurOutputFormat.setOutputPath(job, output); BlurOutputFormat.setIndexLocally(job, false); job.submit(); boolean killCalled = false; while (!job.isComplete()) { Thread.sleep(1000); System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100, job.reduceProgress() * 100); if (job.reduceProgress() > 0.7 && !killCalled) { job.killJob(); killCalled = true; } } assertFalse(job.isSuccessful()); for (int i = 0; i < tableDescriptor.getShardCount(); i++) { Path path = new Path(output, ShardUtil.getShardName(i)); FileSystem fileSystem = path.getFileSystem(job.getConfiguration()); FileStatus[] listStatus = fileSystem.listStatus(path); assertEquals(toString(listStatus), 0, listStatus.length); } }
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
public static boolean runJob(Job job, Progress progress, JobListener jl) throws JobFailedException, JobCancelledException { boolean success = false; if (jl != null) { //append job id to the job name for easy identification job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName()); jl.addJob(job);/*from w w w. ja v a 2s .c om*/ } long start = System.currentTimeMillis(); log.info("Running job {}", job.getJobName()); try { job.submit(); log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); if (progress == null) { job.waitForCompletion(true); } else { float initP = progress.get(); float percentP = 100 - initP; while (job.isComplete() == false) { float p = job.mapProgress() * .9f + job.reduceProgress() * .1f; progress.set(p * percentP + initP); try { Thread.sleep(500); } catch (InterruptedException e) { log.info("Job Cancelled by user"); throw new JobCancelledException("Job Cancelled by user."); } } } log.info("Job {} time: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); if (job.isSuccessful() == false) { throw new JobFailedException("Job failed: " + job.getTrackingURL()); } success = job.isSuccessful(); } catch (InterruptedException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } catch (ClassNotFoundException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } // when submitting jobs under JBoss, Exception doesn't appear to be caught catch (Throwable e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } return success; }
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
/** * Check on the progress of a job and return true if the job has completed. Note * that a return value of true does not mean the job was successful, just that * it completed./* ww w . j a v a 2 s . c o m*/ * * @param job * @param progress * @param jl * @return * @throws IOException * @throws FileNotFoundException * @throws JobFailedException * @throws JobCancelledException */ public static boolean checkJobProgress(Job job, Progress progress, JobListener jl) throws IOException, JobFailedException, JobCancelledException { boolean result = job.isComplete(); if (progress != null) { float initP = progress.get(); float percentP = 100 - initP; if (!result) { float p = job.mapProgress() * .9f + job.reduceProgress() * .1f; progress.set(p * percentP + initP); } } if (result) { if (!job.isSuccessful()) { if (jl != null && jl.isCancelled()) { throw new JobCancelledException(job.getJobName() + " - Job Cancelled by user"); } throw new JobFailedException("Job failed: " + job.getTrackingURL()); } } return result; }
From source file:weka.distributed.hadoop.HadoopJob.java
License:Open Source License
/** * Print status information for the supplied (running) job * /* ww w . j av a2s . c o m*/ * @param job the job to print status info for * @throws IOException if a problem occurs */ protected void printJobStatus(Job job) throws IOException { float setupPercent = job.setupProgress() * 100f; float mapPercent = job.mapProgress() * 100f; float reducePercent = job.reduceProgress() * 100f; String info = getJobName() + " Setup: " + setupPercent + " Map: " + mapPercent + " Reduce: " + reducePercent; statusMessage(info); logMessage(info); }