List of usage examples for org.apache.hadoop.mapreduce Job getJobName
public String getJobName()
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.java
License:Apache License
@Override protected void startUp() throws Exception { // Creates a temporary directory locally for storing all generated files. File tempDir = createTempDirectory(); cleanupTask = createCleanupTask(tempDir); try {//from ww w .jav a 2 s . co m Job job = createJob(new File(tempDir, "mapreduce")); Configuration mapredConf = job.getConfiguration(); classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getPlugins(), context.getPluginInstantiator()); cleanupTask = createCleanupTask(cleanupTask, classLoader); mapredConf.setClassLoader(new WeakReferenceDelegatorClassLoader(classLoader)); ClassLoaders.setContextClassLoader(mapredConf.getClassLoader()); context.setJob(job); beforeSubmit(job); // Localize additional resources that users have requested via BasicMapReduceContext.localize methods Map<String, String> localizedUserResources = localizeUserResources(job, tempDir); // Override user-defined job name, since we set it and depend on the name. // https://issues.cask.co/browse/CDAP-2441 String jobName = job.getJobName(); if (!jobName.isEmpty()) { LOG.warn("Job name {} is being overridden.", jobName); } job.setJobName(getJobName(context)); // Create a temporary location for storing all generated files through the LocationFactory. Location tempLocation = createTempLocationDirectory(); cleanupTask = createCleanupTask(cleanupTask, tempLocation); // For local mode, everything is in the configuration classloader already, hence no need to create new jar if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // After calling beforeSubmit, we know what plugins are needed for the program, hence construct the proper // ClassLoader from here and use it for setting up the job Location pluginArchive = createPluginArchive(tempLocation); if (pluginArchive != null) { job.addCacheArchive(pluginArchive.toURI()); mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName()); } } // set resources for the job TaskType.MAP.setResources(mapredConf, context.getMapperResources()); TaskType.REDUCE.setResources(mapredConf, context.getReducerResources()); // replace user's Mapper & Reducer's with our wrappers in job config MapperWrapper.wrap(job); ReducerWrapper.wrap(job); // packaging job jar which includes cdap classes with dependencies File jobJar = buildJobJar(job, tempDir); job.setJar(jobJar.toURI().toString()); Location programJar = programJarLocation; if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // Copy and localize the program jar in distributed mode programJar = copyProgramJar(tempLocation); job.addCacheFile(programJar.toURI()); List<String> classpath = new ArrayList<>(); // Localize logback.xml Location logbackLocation = createLogbackJar(tempLocation); if (logbackLocation != null) { job.addCacheFile(logbackLocation.toURI()); classpath.add(logbackLocation.getName()); } // Generate and localize the launcher jar to control the classloader of MapReduce containers processes classpath.add("job.jar/lib/*"); classpath.add("job.jar/classes"); Location launcherJar = createLauncherJar( Joiner.on(",").join(MapReduceContainerHelper.getMapReduceClassPath(mapredConf, classpath)), tempLocation); job.addCacheFile(launcherJar.toURI()); // The only thing in the container classpath is the launcher.jar // The MapReduceContainerLauncher inside the launcher.jar will creates a MapReduceClassLoader and launch // the actual MapReduce AM/Task from that // We explicitly localize the mr-framwork, but not use it with the classpath URI frameworkURI = MapReduceContainerHelper.getFrameworkURI(mapredConf); if (frameworkURI != null) { job.addCacheArchive(frameworkURI); } mapredConf.unset(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, launcherJar.getName()); mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, launcherJar.getName()); } MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); // We start long-running tx to be used by mapreduce job tasks. Transaction tx = txClient.startLong(); try { // We remember tx, so that we can re-use it in mapreduce tasks CConfiguration cConfCopy = cConf; contextConfig.set(context, cConfCopy, tx, programJar.toURI(), localizedUserResources); LOG.info("Submitting MapReduce Job: {}", context); // submits job and returns immediately. Shouldn't need to set context ClassLoader. job.submit(); this.job = job; this.transaction = tx; } catch (Throwable t) { Transactions.invalidateQuietly(txClient, tx); throw t; } } catch (Throwable t) { LOG.error("Exception when submitting MapReduce Job: {}", context, t); cleanupTask.run(); throw t; } }
From source file:com.architecting.ch07.MapReduceIndexerTool.java
License:Apache License
private String getJobInfo(Job job) { return "jobName: " + job.getJobName() + ", jobId: " + job.getJobID(); }
From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java
License:Apache License
@Override public boolean run(Job job) throws InterruptedException { JobID jobId = newJobId(new Random().nextInt(Integer.MAX_VALUE)); setJobId(job, jobId);// w w w.ja va2s. com LOG.info(MessageFormat.format("starting job using {0}: {1} ({2})", this, job.getJobID(), job.getJobName())); try { runJob(job); return true; } catch (InterruptedException e) { throw e; } catch (Exception e) { LOG.error(MessageFormat.format("exception was occurred while executing job: {0} ({1})", job.getJobID(), job.getJobName()), e); return false; } }
From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java
License:Apache License
private void runJob(Job job) throws ClassNotFoundException, IOException, InterruptedException { assert job.getJobID() != null; TaskID taskId = newMapTaskId(job.getJobID(), 0); Configuration conf = job.getConfiguration(); OutputFormat<?, ?> output = ReflectionUtils.newInstance(job.getOutputFormatClass(), conf); OutputCommitter committer = output/*w ww . jav a2 s . c o m*/ .getOutputCommitter(newTaskAttemptContext(conf, newTaskAttemptId(taskId, 0))); boolean succeed = false; committer.setupJob(job); try { if (job.getNumReduceTasks() == 0) { runMap(job, null); } else { try (KeyValueSorter<?, ?> sorter = createSorter(job, job.getMapOutputKeyClass(), job.getMapOutputValueClass())) { runMap(job, sorter); runReduce(job, sorter); } } committer.commitJob(job); succeed = true; } finally { if (succeed == false) { try { committer.abortJob(job, State.FAILED); } catch (IOException e) { LOG.error(MessageFormat.format("error occurred while aborting job: {0} ({1})", job.getJobID(), job.getJobName()), e); } } } }
From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" })
private void runReduce(Job job, KeyValueSorter<?, ?> sorter)
throws ClassNotFoundException, IOException, InterruptedException {
Configuration conf = job.getConfiguration();
OutputFormat<?, ?> output = ReflectionUtils.newInstance(job.getOutputFormatClass(), conf);
TaskAttemptID id = newTaskAttemptId(newReduceTaskId(job.getJobID(), 1), 0);
Reducer<?, ?, ?, ?> reducer = ReflectionUtils.newInstance(job.getReducerClass(), conf);
if (LOG.isDebugEnabled()) {
LOG.debug(MessageFormat.format("starting reducer: {0}@{1} ({2}records, {3}bytes)", //$NON-NLS-1$
reducer.getClass().getName(), id, sorter.getRecordCount(), sorter.getSizeInBytes()));
}//from www.jav a2 s.c o m
TaskAttemptContext context = newTaskAttemptContext(conf, id);
OutputCommitter committer = output.getOutputCommitter(context);
committer.setupTask(context);
boolean succeed = false;
try {
ShuffleReader reader = new ShuffleReader(sorter, new Progress());
try {
RecordWriter<?, ?> writer = output.getRecordWriter(newTaskAttemptContext(conf, id));
try {
Reducer.Context c = newReducerContext(conf, id, reader, sorter.getKeyClass(),
sorter.getValueClass(), writer, committer, (RawComparator) job.getGroupingComparator());
reducer.run(c);
} finally {
writer.close(newTaskAttemptContext(conf, id));
}
} finally {
try {
reader.close();
} catch (IOException e) {
LOG.warn(MessageFormat.format("error occurred while reducer mapper input: {0} ({1})", id,
job.getJobName()), e);
}
}
doCommitTask(context, committer);
succeed = true;
} finally {
if (succeed == false) {
doAbortTask(context, committer);
}
}
}
From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java
License:Apache License
private <K, V> KeyValueSorter<?, ?> createSorter(Job job, Class<K> key, Class<V> value) { KeyValueSorter.Options options = getSorterOptions(job.getConfiguration()); if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( "shuffle buffer size: {1}bytes/page, {2}bytes/block, compression:{3} ({0})", //$NON-NLS-1$ job.getJobName(), options.getPageSize(), options.getBlockSize(), options.isCompressBlock())); }/*from w w w . ja v a2 s .c o m*/ return new KeyValueSorter<>(new SerializationFactory(job.getConfiguration()), key, value, job.getSortComparator(), options); }
From source file:com.asakusafw.runtime.stage.AbstractStageClient.java
License:Apache License
private int submit(Job job) throws IOException, InterruptedException, ClassNotFoundException { String jobRunnerClassName = job.getConfiguration().get(StageConstants.PROP_JOB_RUNNER); JobRunner runner = DefaultJobRunner.INSTANCE; if (jobRunnerClassName != null) { Class<?> jobRunnerClass = job.getConfiguration().getClassByName(jobRunnerClassName); runner = (JobRunner) ReflectionUtils.newInstance(jobRunnerClass, job.getConfiguration()); }/*w w w. ja v a 2s. com*/ LOG.info(MessageFormat.format("Submitting Job: {0} (runner: {1})", job.getJobName(), runner)); long start = System.currentTimeMillis(); boolean succeed; if (RuntimeContext.get().isSimulation()) { LOG.info(MessageFormat.format( "Job is skipped because current execution status is in simulation mode: name={0}", job.getJobName())); succeed = true; } else { succeed = runner.run(job); } long end = System.currentTimeMillis(); LOG.info(MessageFormat.format("Job Finished: elapsed=[{3}]ms, succeed={2}, id={0}, name={1}", job.getJobID(), job.getJobName(), succeed, String.valueOf(end - start))); return succeed ? ToolLauncher.JOB_SUCCEEDED : ToolLauncher.JOB_FAILED; }
From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java
License:Apache License
@Override public void configure(Job job) throws IOException, InterruptedException { if (isLocal(job)) { return;// w w w. j a v a 2 s .com } Configuration conf = job.getConfiguration(); long limit = conf.getLong(KEY_LIMIT, -1L); if (limit < 0L) { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Auto stage localize is disabled: {0}", job.getJobName())); } return; } if (isProtected(job)) { return; } long estimated = StageInputDriver.estimateInputSize(job); if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Auto stage localize: job={0}, limit={1}, estimated={2}", job.getJobName(), limit, estimated)); } if (estimated < 0L || estimated > limit) { return; } LOG.info(MessageFormat.format("The job \"{0}\" will run in local mode", job.getJobName())); localize(job); }
From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java
License:Apache License
private void localize(Job job) { Configuration conf = job.getConfiguration(); // reset job-tracker conf.set(KEY_JOBTRACKER, DEFAULT_JOBTRACKER); // replace local directories String tmpDir = conf.get(KEY_TEMPORARY_DIRECTORY, ""); if (tmpDir.isEmpty()) { String name = System.getProperty("user.name", "asakusa"); tmpDir = String.format("/tmp/hadoop-%s/autolocal", name); } else if (tmpDir.length() > 1 && tmpDir.endsWith("/")) { tmpDir = tmpDir.substring(0, tmpDir.length() - 1); }/*from w w w .j a v a2 s. c o m*/ if (conf.getBoolean(KEY_DIRECTORY_QUALIFIER, true)) { String qualifier = UUID.randomUUID().toString(); tmpDir = String.format("%s/%s", tmpDir, qualifier); } LOG.info(MessageFormat.format("Substituting temporary dir: job={0}, target={1}", job.getJobName(), tmpDir)); conf.set(KEY_LOCAL_DIR, tmpDir + "/mapred/local"); conf.set(KEY_STAGING_DIR, tmpDir + "/mapred/staging"); }
From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfigurator.java
License:Apache License
@Override public void configure(Job job) throws IOException, InterruptedException { if (job.getConfiguration().getBoolean(KEY_FORCE, false)) { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("force enabled in-process execution: {0}", //$NON-NLS-1$ job.getJobName())); }// ww w. ja v a 2 s . c o m install(job); return; } long limit = job.getConfiguration().getLong(KEY_LIMIT, -1L); if (limit < 0L) { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("in-process execution is disabled: {0}", //$NON-NLS-1$ job.getJobName())); } return; } if (hasCustomJobRunner(job)) { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("custom job runner is already activated: {0}", //$NON-NLS-1$ job.getJobName())); } return; } long estimated = getEstimatedJobSize(job); if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("estimated input data size for in-process execution: " //$NON-NLS-1$ + "job={0}, limit={1}, estimated={2}", //$NON-NLS-1$ job.getJobName(), limit, estimated)); } if (estimated < 0L || estimated > limit) { return; } if (LOG.isInfoEnabled()) { LOG.info(MessageFormat.format("enable in-process execution: job={0}, limit={1}, estimated={2}", job.getJobName(), limit, estimated)); } install(job); }