List of usage examples for org.apache.hadoop.mapreduce Job getJobName
public String getJobName()
From source file:org.janusgraph.hadoop.config.HBaseAuthHelper.java
License:Apache License
public static void setHBaseAuthToken(Configuration configuration, Job job) throws IOException { // Get HBase authentication token (when configured) String hbaseAuthentication = configuration.get("hbase.security.authentication"); if (null != hbaseAuthentication && hbaseAuthentication.equals("kerberos")) { String quorumCfgKey = "hbase.zookeeper.quorum"; log.info("Obtaining HBase Auth Token from ZooKeeper quorum " + configuration.get(quorumCfgKey)); final String className = "org.apache.hadoop.hbase.security.User"; try {// w w w .j a v a2 s . c o m Class<?> clazz = HBaseAuthHelper.class.getClassLoader().loadClass(className); Method getCurrent = clazz.getMethod("getCurrent"); Object user = getCurrent.invoke(null); Method obtainAuthTokenForJob = clazz.getMethod("obtainAuthTokenForJob", Configuration.class, Job.class); obtainAuthTokenForJob.invoke(user, configuration, job); log.info("Obtained HBase Auth Token from ZooKeeper quorum {} for job {}", configuration.get(quorumCfgKey), job.getJobName()); } catch (Throwable t) { log.error("Failed to generate or store HBase auth token", t); } } else { log.info("Not obtaining HBase Auth Token for MapReduce job " + job.getJobName()); } }
From source file:org.kiji.mapreduce.framework.JobHistoryKijiTable.java
License:Apache License
/** * Writes a job into the JobHistoryKijiTable. * * @param job The job to save./*from w w w .jav a 2s .c o m*/ * @param startTime The time the job began, in milliseconds. * @param endTime The time the job ended, in milliseconds * @throws IOException If there is an error writing to the table. */ public void recordJob(Job job, long startTime, long endTime) throws IOException { KijiTableWriter writer = mKijiTable.openTableWriter(); EntityId jobEntity = mKijiTable.getEntityId(job.getJobID().toString()); try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_ID_QUALIFIER, startTime, job.getJobID().toString()); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_NAME_QUALIFIER, startTime, job.getJobName()); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_START_TIME_QUALIFIER, startTime, startTime); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_END_TIME_QUALIFIER, startTime, endTime); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_END_STATUS_QUALIFIER, startTime, job.isSuccessful() ? "SUCCEEDED" : "FAILED"); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_COUNTERS_QUALIFIER, startTime, job.getCounters().toString()); job.getConfiguration().writeXml(baos); writer.put(jobEntity, JOB_HISTORY_FAMILY, JOB_HISTORY_CONFIGURATION_QUALIFIER, startTime, baos.toString("UTF-8")); writeIndividualCounters(writer, job); } finally { ResourceUtils.closeOrLog(writer); } }
From source file:org.kiji.mapreduce.JobHistoryKijiTable.java
License:Apache License
/** * Writes a job into the JobHistoryKijiTable. * * @param job The job to save.//from w w w. j a v a2 s . c o m * @param startTime The time the job began, in milliseconds. * @param endTime The time the job ended, in milliseconds * @throws IOException If there is an error writing to the table. */ public void recordJob(Job job, long startTime, long endTime) throws IOException { KijiTableWriter writer = mKijiTable.openTableWriter(); EntityId jobEntity = mKijiTable.getEntityId(job.getJobID().toString()); try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); writer.put(jobEntity, "info", "jobId", startTime, job.getJobID().toString()); writer.put(jobEntity, "info", "jobName", startTime, job.getJobName()); writer.put(jobEntity, "info", "startTime", startTime, startTime); writer.put(jobEntity, "info", "endTime", startTime, endTime); writer.put(jobEntity, "info", "counters", startTime, job.getCounters().toString()); job.getConfiguration().writeXml(baos); writer.put(jobEntity, "info", "configuration", startTime, baos.toString("UTF-8")); } finally { IOUtils.closeQuietly(writer); } }
From source file:org.mrgeo.data.accumulo.input.image.AccumuloMrsImagePyramidInputFormatProvider.java
License:Apache License
@Override public void setupJob(Job job, final Properties providerProperties) throws DataProviderException { super.setupJob(job, providerProperties); //zoomLevelsInPyramid = new ArrayList<Integer>(); log.info("Setting up job " + job.getJobName()); // lets look into the properties coming in if (providerProperties != null) { Set<Object> k1 = providerProperties.keySet(); ArrayList<String> k2 = new ArrayList<String>(); for (Object o : k1) { k2.add(o.toString());/* www . j a va 2 s. c o m*/ } Collections.sort(k2); for (int x = 0; x < k2.size(); x++) { log.info("provider property " + x + ": k='" + k2.get(x) + "' v='" + providerProperties.getProperty(k2.get(x)) + "'"); } } // set the needed information if (props == null) { props = new Properties(); props.putAll(AccumuloConnector.getAccumuloProperties()); } String connUser = props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER); log.info("connecting to accumulo as user " + connUser); if (providerProperties != null) { props.putAll(providerProperties); } if (props.size() == 0) { throw new RuntimeException("No configuration for Accumulo!"); } // just in case this gets overwritten for (String k : MrGeoAccumuloConstants.MRGEO_ACC_KEYS_CONNECTION) { job.getConfiguration().set(k, props.getProperty(k)); } for (String k : MrGeoAccumuloConstants.MRGEO_ACC_KEYS_DATA) { job.getConfiguration().set(k, props.getProperty(k)); } if (props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_OUTPUT_TABLE) == null) { job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_OUTPUT_TABLE, this.table); } else { job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_OUTPUT_TABLE, props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_OUTPUT_TABLE)); } // make sure the password is set with Base64Encoding String pw = props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD); String isEnc = props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PWENCODED64, "false"); String pwDec = pw; if (isEnc.equalsIgnoreCase("true")) { job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD, props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD)); pwDec = new String(Base64.decodeBase64(pw.getBytes())); job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD, pwDec); } else { byte[] p = Base64 .encodeBase64(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD).getBytes()); job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD, new String(p)); job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PWENCODED64, new String("true")); } // get the visualizations if (job.getConfiguration().get("protectionLevel") != null) { job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_VIZ, job.getConfiguration().get("protectionLevel")); } // if(props.containsKey(MrGeoAccumuloConstants.MRGEO_ACC_KEY_VIZ)){ // job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_VIZ, // props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_VIZ)); // } if (props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_AUTHS) != null) { auths = new Authorizations(props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_AUTHS).split(",")); } else { auths = new Authorizations(); } String enc = AccumuloConnector.encodeAccumuloProperties(context.getFirstInput()); job.getConfiguration().set(MrGeoAccumuloConstants.MRGEO_ACC_KEY_ENCODED, enc); //job.setInputFormatClass(AccumuloMrsImagePyramidInputFormat.class); // check for base64 encoded password // AccumuloMrsImagePyramidInputFormat.setInputInfo(job.getConfiguration(), // props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER), // pwDec.getBytes(), // table, // auths); AccumuloMrsImagePyramidInputFormat.setZooKeeperInstance(job, props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_INSTANCE), props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_ZOOKEEPERS)); PasswordToken pt = new PasswordToken( //job.getConfiguration().get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD) pwDec); log.info("connecting to accumulo with user " + connUser); //log.info("password used to connect is " + job.getConfiguration().get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_PASSWORD)); log.info("password (decoded) used to connect is " + pwDec); log.info("scan authorizations are " + auths); log.info("authorizations from config = " + job.getConfiguration().get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_AUTHS)); try { AccumuloMrsImagePyramidInputFormat.setConnectorInfo(job, connUser, //props.getProperty(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER), //job.getConfiguration().get(MrGeoAccumuloConstants.MRGEO_ACC_KEY_USER), pt); } catch (AccumuloSecurityException ase) { log.info("problem with authentication elements."); return; } AccumuloInputFormat.setScanAuthorizations(job, auths); // get the input table for (final String input : context.getInputs()) { // put encoded string for Accumulo connections //TODO what needs to be done here? log.info("working with source " + input + " with auths = " + auths); AccumuloMrsImagePyramidInputFormat.setInputTableName(job, input); } // end for loop log.info("setting column family to regex " + Integer.toString(context.getZoomLevel())); // think about scanners - set the zoom level of the job IteratorSetting regex = new IteratorSetting(51, "regex", RegExFilter.class); RegExFilter.setRegexs(regex, null, Integer.toString(context.getZoomLevel()), null, null, false); Collection<Pair<Text, Text>> colFamColQual = new ArrayList<Pair<Text, Text>>(); Pair<Text, Text> p1 = new Pair<Text, Text>(new Text(Integer.toString(context.getZoomLevel())), null); colFamColQual.add(p1); AccumuloMrsImagePyramidInputFormat.fetchColumns(job, colFamColQual); AccumuloMrsImagePyramidInputFormat.addIterator(job, regex); //job.setJarByClass(this.getClass()); String cp = job.getConfiguration().get("mapred.job.classpath.files"); log.info("mapred.job.classpath.files = " + cp); }
From source file:org.mrgeo.data.accumulo.input.image.AccumuloMrsPyramidInputFormatProvider.java
License:Apache License
@Override public void setupJob(Job job, final MrsImageDataProvider provider) throws DataProviderException { super.setupJob(job, provider); log.info("Setting up job " + job.getJobName()); setupConfig(job, provider);/*from www. j a va 2s . c o m*/ }
From source file:org.mrgeo.mapreduce.job.JobListener.java
License:Apache License
public boolean cancelAll() throws JobCancelFailedException { boolean success = true; setCancelled();//w ww .j a va 2 s. c o m synchronized (jobsListLock) { for (Job job : jobsList) { _log.info("User requested cancellation - killing job " + job.getJobName()); //this is a hadoop job, so kill it. try { job.killJob(); } catch (IOException e) { //log it, make a note of the fact that the job cancel failed //so you can propagate the exception back _log.error("Kill job failed for " + job.getJobID()); success = false; } } if (!success) { throw new JobCancelFailedException( "Cancel failed for some of the hadoop jobs, see log for details."); } } return success; }
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
public static boolean runJob(Job job, Progress progress, JobListener jl) throws JobFailedException, JobCancelledException { boolean success = false; if (jl != null) { //append job id to the job name for easy identification job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName()); jl.addJob(job);// www . j av a 2 s . c o m } long start = System.currentTimeMillis(); log.info("Running job {}", job.getJobName()); try { job.submit(); log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); if (progress == null) { job.waitForCompletion(true); } else { float initP = progress.get(); float percentP = 100 - initP; while (job.isComplete() == false) { float p = job.mapProgress() * .9f + job.reduceProgress() * .1f; progress.set(p * percentP + initP); try { Thread.sleep(500); } catch (InterruptedException e) { log.info("Job Cancelled by user"); throw new JobCancelledException("Job Cancelled by user."); } } } log.info("Job {} time: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); if (job.isSuccessful() == false) { throw new JobFailedException("Job failed: " + job.getTrackingURL()); } success = job.isSuccessful(); } catch (InterruptedException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } catch (ClassNotFoundException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } // when submitting jobs under JBoss, Exception doesn't appear to be caught catch (Throwable e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } return success; }
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
public static void runJobAsynchronously(Job job, JobListener jl) throws IOException, JobFailedException, JobCancelledException { if (jl != null) { //append job id to the job name for easy identification job.setJobName("ID_" + jl.getUserJobId() + "_" + job.getJobName()); jl.addJob(job);/*from w w w . j a v a 2s .com*/ } try { long start = System.currentTimeMillis(); log.info("Running asynchronous job {}", job.getJobName()); job.submit(); log.info("Job {} startup: {}ms", job.getJobName(), (System.currentTimeMillis() - start)); } catch (InterruptedException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } catch (ClassNotFoundException e) { e.printStackTrace(); throw new JobFailedException(e.getMessage()); } }
From source file:org.mrgeo.mapreduce.MapReduceUtils.java
License:Apache License
/** * Check on the progress of a job and return true if the job has completed. Note * that a return value of true does not mean the job was successful, just that * it completed./*from w w w . jav a2 s . c om*/ * * @param job * @param progress * @param jl * @return * @throws IOException * @throws FileNotFoundException * @throws JobFailedException * @throws JobCancelledException */ public static boolean checkJobProgress(Job job, Progress progress, JobListener jl) throws IOException, JobFailedException, JobCancelledException { boolean result = job.isComplete(); if (progress != null) { float initP = progress.get(); float percentP = 100 - initP; if (!result) { float p = job.mapProgress() * .9f + job.reduceProgress() * .1f; progress.set(p * percentP + initP); } } if (result) { if (!job.isSuccessful()) { if (jl != null && jl.isCancelled()) { throw new JobCancelledException(job.getJobName() + " - Job Cancelled by user"); } throw new JobFailedException("Job failed: " + job.getTrackingURL()); } } return result; }
From source file:org.oclc.firefly.hadoop.backup.Backup.java
License:Apache License
/** * Performs a complete copy of the source hbase to the given destination * @param tables The names of the tables to backup * @param maxTries The maximum number of times to try to copy regions. * @return True if successful, false otherwise * @throws IOException If failed to interact with Hadoop * @throws ClassNotFoundException /*from w w w. jav a 2s . c om*/ * @throws InterruptedException */ public boolean doMajorCopy(String[] tables, int maxTries) throws IOException, InterruptedException, ClassNotFoundException { boolean ret = false; String username = getUsername(); short replication = (short) getInitialReplication(); // Get a list of regions from HBase // Then filter out the regions we are not extracting, and group them by table List<CatalogRow> regions = getHBaseRegions(srcConf); Map<String, List<CatalogRow>> filtered = groupAndFilter(regions, tables); List<Pair<String, HRegionInfo>> mapperInput = new ArrayList<Pair<String, HRegionInfo>>(); // Prepare the input for the mappers to use // This creates a list of region server and region pairs LOG.info("Exporting the following tables:"); for (Entry<String, List<CatalogRow>> entry : filtered.entrySet()) { String tablename = entry.getKey(); List<CatalogRow> rows = entry.getValue(); LOG.info(". " + tablename); for (CatalogRow r : rows) { String regionServer = r.getHost() + ":" + r.getPort(); HRegionInfo region = r.getHRegionInfo(); mapperInput.add(Pair.newPair(regionServer, region)); } } // Make sure we write to a directory that does not exist backupDirectoryPath = createBackupDirectory(getCurrentDateString()); LOG.info("Starting backup path: " + backupDirectoryPath); // Copy the .tableinfo files for the tables we are extracting // These files are not copied by the MR job as it only focuses on regions List<FileStatus> tableInfoFiles = getTableInfoFiles(srcFs, filtered); for (FileStatus file : tableInfoFiles) { Path srcFilePath = file.getPath(); Path relPath = new Path(BackupUtils.getFsRelativePath(srcFs, srcFilePath)); Path dstFilePath = new Path(backupDirectoryPath.toString() + relPath.toString()); BackupUtils.copy(srcFs, srcFilePath, dstFs, dstFilePath, buffer, username, replication); } // Dispatch MR job and monitor // Retry regions if necessary if (mapperInput.size() > 0) { int tries = 0; while (!ret && (maxTries == 0 || tries < maxTries)) { if (getNumMapTasks() > mapperInput.size()) { setNumMapTasks(mapperInput.size()); LOG.info("Not enough regions. Reducing number of map tasks"); } // Generate a list of mapper input files and create job List<Path> sourceFiles = createMapperInputSequenceFiles(mapperInput, getNumMapTasks(), srcFs, tries); Job job = createMRJob(srcConf, dstConf, sourceFiles, backupDirectoryPath, tries); LOG.info(job.getJobName()); LOG.info("--------------------------------------------------"); LOG.info("Number of regions : " + mapperInput.size()); LOG.info("Number of map tasks: " + getNumMapTasks()); LOG.info("Mapper input path : " + getMapInputDirectory(tries)); LOG.info("Mapper output path : " + FileOutputFormat.getOutputPath(job)); LOG.info("--------------------------------------------------"); job.waitForCompletion(true); if (job.isSuccessful()) { // Check if any regions failed Counters counters = job.getCounters(); Counter failedCounter = counters.findCounter("Backup", "FailedRegions"); long failed = failedCounter.getValue(); if (failed > 0) { LOG.info("Number of failed regions: " + failed + "."); // get a fresh list of regions to copy List<Pair<String, HRegionInfo>> failedRegions = getFailedRegions(srcFs, srcConf, tries); addCopiedRegions(mapperInput, failedRegions); mapperInput = getRemainingRegions(mapperInput, tables); for (Pair<String, HRegionInfo> pair : mapperInput) { LOG.info("Retry: " + pair.getSecond()); } if (mapperInput.size() == 0) { ret = true; backupDirectoryPath = appendEndTime(backupDirectoryPath); LOG.warn("No regions left to copy, but expected to copy more. " + "Please inspect logs/files manually for errors"); } } else { ret = true; addCopiedRegions(mapperInput, null); backupDirectoryPath = appendEndTime(backupDirectoryPath); LOG.info("MR job finished successfully"); } } else { LOG.error("An unexpected error occurred during the MR job. Please see MR logs."); break; } tries++; } if (ret) { if (verifyCopiedRegions()) { LOG.info("Verification passed succesfully"); } else { ret = false; LOG.info("Verification failed. Please inspect errors manually"); } } else { LOG.info("No attempts left. Try setting -n to a higher value, or setting it to 0"); } } if (ret) { // Set replication factor of backup directory to default. // This may not be the best solution, but let built-in shell take care of it // because it can do it recursively with out us having to rediscover all the files short finalReplication = (short) getFinalReplication(); if (replication != finalReplication) { FsShell shell = new FsShell(dstConf); String[] repArgs = { "-setrep", "-R", "-w", "" + finalReplication, backupDirectoryPath.toString() }; try { LOG.info("Setting final replication factor of backup files to " + finalReplication); shell.run(repArgs); } catch (Exception e) { LOG.warn("Could not set replication factor of backup files to " + finalReplication); } } } return ret; }