List of usage examples for org.apache.hadoop.mapreduce Job isSuccessful
public boolean isSuccessful() throws IOException
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
public static boolean runJob(Job job, Progress progress, JobListener jl) throws JobFailedException, JobCancelledException { boolean success = false; if (jl != null) { //append job id to the job name for easy identification job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName()); jl.addJob(job);/*from www . j a va2 s.c o m*/ } long start = System.currentTimeMillis(); log.info("Running job {}", job.getJobName()); try { job.submit(); log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); if (progress == null) { job.waitForCompletion(true); } else { float initP = progress.get(); float percentP = 100 - initP; while (job.isComplete() == false) { float p = job.mapProgress() * .9f + job.reduceProgress() * .1f; progress.set(p * percentP + initP); try { Thread.sleep(500); } catch (InterruptedException e) { log.info("Job Cancelled by user"); throw new JobCancelledException("Job Cancelled by user."); } } } log.info("Job {} time: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); if (job.isSuccessful() == false) { throw new JobFailedException("Job failed: " + job.getTrackingURL()); } success = job.isSuccessful(); } catch (InterruptedException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } catch (ClassNotFoundException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } // when submitting jobs under JBoss, Exception doesn't appear to be caught catch (Throwable e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } return success; }
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
/** * Check on the progress of a job and return true if the job has completed. Note * that a return value of true does not mean the job was successful, just that * it completed./*from w w w . ja v a2 s . com*/ * * @param job * @param progress * @param jl * @return * @throws IOException * @throws FileNotFoundException * @throws JobFailedException * @throws JobCancelledException */ public static boolean checkJobProgress(Job job, Progress progress, JobListener jl) throws IOException, JobFailedException, JobCancelledException { boolean result = job.isComplete(); if (progress != null) { float initP = progress.get(); float percentP = 100 - initP; if (!result) { float p = job.mapProgress() * .9f + job.reduceProgress() * .1f; progress.set(p * percentP + initP); } } if (result) { if (!job.isSuccessful()) { if (jl != null && jl.isCancelled()) { throw new JobCancelledException(job.getJobName() + " - Job Cancelled by user"); } throw new JobFailedException("Job failed: " + job.getTrackingURL()); } } return result; }
From source file:org.oclc.firefly.hadoop.backup.Backup.java
License:Apache License
/** * Performs a complete copy of the source hbase to the given destination * @param tables The names of the tables to backup * @param maxTries The maximum number of times to try to copy regions. * @return True if successful, false otherwise * @throws IOException If failed to interact with Hadoop * @throws ClassNotFoundException /*from ww w . j av a 2 s . com*/ * @throws InterruptedException */ public boolean doMajorCopy(String[] tables, int maxTries) throws IOException, InterruptedException, ClassNotFoundException { boolean ret = false; String username = getUsername(); short replication = (short) getInitialReplication(); // Get a list of regions from HBase // Then filter out the regions we are not extracting, and group them by table List<CatalogRow> regions = getHBaseRegions(srcConf); Map<String, List<CatalogRow>> filtered = groupAndFilter(regions, tables); List<Pair<String, HRegionInfo>> mapperInput = new ArrayList<Pair<String, HRegionInfo>>(); // Prepare the input for the mappers to use // This creates a list of region server and region pairs LOG.info("Exporting the following tables:"); for (Entry<String, List<CatalogRow>> entry : filtered.entrySet()) { String tablename = entry.getKey(); List<CatalogRow> rows = entry.getValue(); LOG.info(". " + tablename); for (CatalogRow r : rows) { String regionServer = r.getHost() + ":" + r.getPort(); HRegionInfo region = r.getHRegionInfo(); mapperInput.add(Pair.newPair(regionServer, region)); } } // Make sure we write to a directory that does not exist backupDirectoryPath = createBackupDirectory(getCurrentDateString()); LOG.info("Starting backup path: " + backupDirectoryPath); // Copy the .tableinfo files for the tables we are extracting // These files are not copied by the MR job as it only focuses on regions List<FileStatus> tableInfoFiles = getTableInfoFiles(srcFs, filtered); for (FileStatus file : tableInfoFiles) { Path srcFilePath = file.getPath(); Path relPath = new Path(BackupUtils.getFsRelativePath(srcFs, srcFilePath)); Path dstFilePath = new Path(backupDirectoryPath.toString() + relPath.toString()); BackupUtils.copy(srcFs, srcFilePath, dstFs, dstFilePath, buffer, username, replication); } // Dispatch MR job and monitor // Retry regions if necessary if (mapperInput.size() > 0) { int tries = 0; while (!ret && (maxTries == 0 || tries < maxTries)) { if (getNumMapTasks() > mapperInput.size()) { setNumMapTasks(mapperInput.size()); LOG.info("Not enough regions. Reducing number of map tasks"); } // Generate a list of mapper input files and create job List<Path> sourceFiles = createMapperInputSequenceFiles(mapperInput, getNumMapTasks(), srcFs, tries); Job job = createMRJob(srcConf, dstConf, sourceFiles, backupDirectoryPath, tries); LOG.info(job.getJobName()); LOG.info("--------------------------------------------------"); LOG.info("Number of regions : " + mapperInput.size()); LOG.info("Number of map tasks: " + getNumMapTasks()); LOG.info("Mapper input path : " + getMapInputDirectory(tries)); LOG.info("Mapper output path : " + FileOutputFormat.getOutputPath(job)); LOG.info("--------------------------------------------------"); job.waitForCompletion(true); if (job.isSuccessful()) { // Check if any regions failed Counters counters = job.getCounters(); Counter failedCounter = counters.findCounter("Backup", "FailedRegions"); long failed = failedCounter.getValue(); if (failed > 0) { LOG.info("Number of failed regions: " + failed + "."); // get a fresh list of regions to copy List<Pair<String, HRegionInfo>> failedRegions = getFailedRegions(srcFs, srcConf, tries); addCopiedRegions(mapperInput, failedRegions); mapperInput = getRemainingRegions(mapperInput, tables); for (Pair<String, HRegionInfo> pair : mapperInput) { LOG.info("Retry: " + pair.getSecond()); } if (mapperInput.size() == 0) { ret = true; backupDirectoryPath = appendEndTime(backupDirectoryPath); LOG.warn("No regions left to copy, but expected to copy more. " + "Please inspect logs/files manually for errors"); } } else { ret = true; addCopiedRegions(mapperInput, null); backupDirectoryPath = appendEndTime(backupDirectoryPath); LOG.info("MR job finished successfully"); } } else { LOG.error("An unexpected error occurred during the MR job. Please see MR logs."); break; } tries++; } if (ret) { if (verifyCopiedRegions()) { LOG.info("Verification passed succesfully"); } else { ret = false; LOG.info("Verification failed. Please inspect errors manually"); } } else { LOG.info("No attempts left. Try setting -n to a higher value, or setting it to 0"); } } if (ret) { // Set replication factor of backup directory to default. // This may not be the best solution, but let built-in shell take care of it // because it can do it recursively with out us having to rediscover all the files short finalReplication = (short) getFinalReplication(); if (replication != finalReplication) { FsShell shell = new FsShell(dstConf); String[] repArgs = { "-setrep", "-R", "-w", "" + finalReplication, backupDirectoryPath.toString() }; try { LOG.info("Setting final replication factor of backup files to " + finalReplication); shell.run(repArgs); } catch (Exception e) { LOG.warn("Could not set replication factor of backup files to " + finalReplication); } } } return ret; }
From source file:org.trend.hgraph.test.AbstractHBaseMiniClusterTest.java
License:Apache License
/** * Import local file to table./*from w ww .ja v a 2 s .c o m*/ * @param conf * @param args * @param TABLE_NAME * @param INPUT_FILE * @throws IOException * @throws InterruptedException * @throws ClassNotFoundException */ protected static void importLocalFile2Table(final Configuration conf, final String[] args, final String TABLE_NAME, final String INPUT_FILE) throws IOException, InterruptedException, ClassNotFoundException { Validate.notEmpty(INPUT_FILE, "INPUT_FILE shall not be empty or null"); InputStream ips = ClassLoader.getSystemResourceAsStream(INPUT_FILE); assertNotNull(ips); FileSystem fs = FileSystem.get(conf); FSDataOutputStream op = fs.create(new Path(INPUT_FILE), true); IOUtils.write(IOUtils.toString(ips), op, HConstants.UTF8_ENCODING); IOUtils.closeQuietly(op); IOUtils.closeQuietly(ips); int length = args.length + 2; String[] newArgs = new String[length]; System.arraycopy(args, 0, newArgs, 0, args.length); newArgs[length - 2] = TABLE_NAME; // newArgs[length - 1] = INPUT_FILE_PATH + INPUT_FILE; newArgs[length - 1] = INPUT_FILE; Job job = ImportTsv.createSubmittableJob(conf, newArgs); job.waitForCompletion(true); assertTrue(job.isSuccessful()); }
From source file:pl.edu.icm.coansys.richimporttsv.jobs.mapreduce.TestRichImportTsv.java
License:Apache License
private HTable doMROnTableTest(String inputFile, String family, String tableName, String line, String[] args) throws Exception { GenericOptionsParser opts = new GenericOptionsParser(UTIL.getConfiguration(), args); Configuration config = UTIL.getConfiguration(); args = opts.getRemainingArgs();/*w ww .j a v a2s . co m*/ FileSystem fs = UTIL.getDFSCluster().getFileSystem(); FSDataOutputStream op = fs.create(new Path(inputFile), true); op.write(line.getBytes(HConstants.UTF8_ENCODING)); op.close(); assertTrue(fs.exists(new Path(inputFile))); final byte[] FAM = Bytes.toBytes(family); final byte[] TAB = Bytes.toBytes(tableName); HTable htableImport = UTIL.createTable(TAB, FAM); assertEquals(0, UTIL.countRows(htableImport)); Job job = RichImportTsv.createSubmittableJob(config, args); job.waitForCompletion(false); assertTrue(job.isSuccessful()); return htableImport; }
From source file:reconcile.hbase.mapreduce.ChainableAnnotationJob.java
License:Open Source License
@Override public int run(String[] args) throws Exception { conf = HBaseConfiguration.create();//from w ww . j ava2 s. com // important to switch spec exec off. // We don't want to have something duplicated for perfomance reasons. conf.set("mapred.map.tasks.speculative.execution", "false"); // since our parse takes so long, we don't want to cache rows -- the scanner might time out conf.set("hbase.client.scanner.caching", "1"); JobConfig jobConfig = new JobConfig(args); Scan scan = new Scan(); int status = 0; try { LOG.info("Before map/reduce startup"); Job job = new Job(conf, getClass().getSimpleName()); job.setJarByClass(this.getClass()); init(jobConfig, job, scan); jobConfig.initTableMapperNoReducer(LOG, job, scan, getMapperClass()); LOG.info("Started "); job.waitForCompletion(true); if (!job.isSuccessful()) status = 1; LOG.info("After map/reduce completion"); finish(); } catch (Exception e) { e.printStackTrace(); status = 1; } LOG.info("Return run status(0=success,1=failure)(" + status + ")"); return status; }
From source file:test.SomeToolCopy.java
License:Apache License
@Override public int run(String[] args) throws Exception { Job j = new SomeToolCopy().createJob(getConf()); j.waitForCompletion(true);/*from w w w . java 2s. c o m*/ assertTrue("Job failed ", j.isSuccessful()); return Integer.valueOf(args[0]); }
From source file:weka.distributed.hadoop.HadoopJob.java
License:Open Source License
/** * Runs the supplied job// w w w .ja v a 2 s . c o m * * @param job the job to run * @return true if the job was successful * @throws DistributedWekaException if a problem occurs */ protected boolean runJob(Job job) throws DistributedWekaException { try { m_stopRunningJob = false; if (DistributedJobConfig.isEmpty(getLoggingInterval())) { m_loggingInterval = "10"; } int logInterval = Integer.parseInt(m_loggingInterval); System.out.println("Setting logging interval to " + logInterval); job.submit(); try { int taskCompletionEventIndex = 0; while (!m_stopRunningJob && !job.isComplete()) { if (logInterval >= 1) { printJobStatus(job); taskCompletionEventIndex += logTaskMessages(job, taskCompletionEventIndex); Thread.sleep(logInterval * 1000); } else { Thread.sleep(60000); } } } catch (InterruptedException ie) { logMessage(ie.getMessage()); m_stopRunningJob = true; } if (m_stopRunningJob && !job.isComplete()) { job.killJob(); } m_stopRunningJob = false; return job.isSuccessful(); } catch (Exception ex) { throw new DistributedWekaException(ex); } }