List of usage examples for org.apache.hadoop.conf Configuration setClassLoader
public void setClassLoader(ClassLoader classLoader)
From source file:azkaban.web.pages.HdfsBrowserServlet.java
License:Apache License
@Override public void init(ServletConfig config) throws ServletException { super.init(config); try {//w ww. j a v a 2 s .co m Configuration conf = new Configuration(); conf.setClassLoader(this.getApplication().getClassLoader()); _fs = FileSystem.get(conf); } catch (IOException e) { throw new ServletException(e); } }
From source file:co.cask.cdap.app.runtime.spark.SparkRuntimeUtils.java
License:Apache License
/** * Sets the context ClassLoader to the given {@link SparkClassLoader}. It will also set the * ClassLoader for the {@link Configuration} contained inside the {@link SparkClassLoader}. * * @return a {@link Cancellable} to reset the classloader to the one prior to the call *///from w ww . j a va 2 s . c o m public static Cancellable setContextClassLoader(final SparkClassLoader sparkClassLoader) { final Configuration hConf = sparkClassLoader.getRuntimeContext().getConfiguration(); final ClassLoader oldConfClassLoader = hConf.getClassLoader(); // Always wrap it with WeakReference to avoid ClassLoader leakage from Spark. ClassLoader classLoader = new WeakReferenceDelegatorClassLoader(sparkClassLoader); hConf.setClassLoader(classLoader); final ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(classLoader); return new Cancellable() { @Override public void cancel() { hConf.setClassLoader(oldConfClassLoader); ClassLoaders.setContextClassLoader(oldClassLoader); // Do not remove the next line. // This is necessary to keep a strong reference to the SparkClassLoader so that it won't get GC until this // cancel() is called LOG.trace("Reset context ClassLoader. The SparkClassLoader is: {}", sparkClassLoader); } }; }
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.java
License:Apache License
@Override protected void startUp() throws Exception { // Creates a temporary directory locally for storing all generated files. File tempDir = createTempDirectory(); cleanupTask = createCleanupTask(tempDir); try {//from ww w. j a va 2 s . c o m Job job = createJob(new File(tempDir, "mapreduce")); Configuration mapredConf = job.getConfiguration(); classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getPlugins(), context.getPluginInstantiator()); cleanupTask = createCleanupTask(cleanupTask, classLoader); mapredConf.setClassLoader(new WeakReferenceDelegatorClassLoader(classLoader)); ClassLoaders.setContextClassLoader(mapredConf.getClassLoader()); context.setJob(job); beforeSubmit(job); // Localize additional resources that users have requested via BasicMapReduceContext.localize methods Map<String, String> localizedUserResources = localizeUserResources(job, tempDir); // Override user-defined job name, since we set it and depend on the name. // https://issues.cask.co/browse/CDAP-2441 String jobName = job.getJobName(); if (!jobName.isEmpty()) { LOG.warn("Job name {} is being overridden.", jobName); } job.setJobName(getJobName(context)); // Create a temporary location for storing all generated files through the LocationFactory. Location tempLocation = createTempLocationDirectory(); cleanupTask = createCleanupTask(cleanupTask, tempLocation); // For local mode, everything is in the configuration classloader already, hence no need to create new jar if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // After calling beforeSubmit, we know what plugins are needed for the program, hence construct the proper // ClassLoader from here and use it for setting up the job Location pluginArchive = createPluginArchive(tempLocation); if (pluginArchive != null) { job.addCacheArchive(pluginArchive.toURI()); mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName()); } } // set resources for the job TaskType.MAP.setResources(mapredConf, context.getMapperResources()); TaskType.REDUCE.setResources(mapredConf, context.getReducerResources()); // replace user's Mapper & Reducer's with our wrappers in job config MapperWrapper.wrap(job); ReducerWrapper.wrap(job); // packaging job jar which includes cdap classes with dependencies File jobJar = buildJobJar(job, tempDir); job.setJar(jobJar.toURI().toString()); Location programJar = programJarLocation; if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // Copy and localize the program jar in distributed mode programJar = copyProgramJar(tempLocation); job.addCacheFile(programJar.toURI()); List<String> classpath = new ArrayList<>(); // Localize logback.xml Location logbackLocation = createLogbackJar(tempLocation); if (logbackLocation != null) { job.addCacheFile(logbackLocation.toURI()); classpath.add(logbackLocation.getName()); } // Generate and localize the launcher jar to control the classloader of MapReduce containers processes classpath.add("job.jar/lib/*"); classpath.add("job.jar/classes"); Location launcherJar = createLauncherJar( Joiner.on(",").join(MapReduceContainerHelper.getMapReduceClassPath(mapredConf, classpath)), tempLocation); job.addCacheFile(launcherJar.toURI()); // The only thing in the container classpath is the launcher.jar // The MapReduceContainerLauncher inside the launcher.jar will creates a MapReduceClassLoader and launch // the actual MapReduce AM/Task from that // We explicitly localize the mr-framwork, but not use it with the classpath URI frameworkURI = MapReduceContainerHelper.getFrameworkURI(mapredConf); if (frameworkURI != null) { job.addCacheArchive(frameworkURI); } mapredConf.unset(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, launcherJar.getName()); mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, launcherJar.getName()); } MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); // We start long-running tx to be used by mapreduce job tasks. Transaction tx = txClient.startLong(); try { // We remember tx, so that we can re-use it in mapreduce tasks CConfiguration cConfCopy = cConf; contextConfig.set(context, cConfCopy, tx, programJar.toURI(), localizedUserResources); LOG.info("Submitting MapReduce Job: {}", context); // submits job and returns immediately. Shouldn't need to set context ClassLoader. job.submit(); this.job = job; this.transaction = tx; } catch (Throwable t) { Transactions.invalidateQuietly(txClient, tx); throw t; } } catch (Throwable t) { LOG.error("Exception when submitting MapReduce Job: {}", context, t); cleanupTask.run(); throw t; } }
From source file:com.armon.test.quartz.QuartzConfiguration.java
License:Apache License
/** * Creates a Configuration with quartz resources * @return a Configuration with quartz resources *//* w w w . j a va 2 s . c o m*/ public static Configuration create() { Configuration conf = new Configuration(); // In case QuartzConfiguration is loaded from a different classloader than // Configuration, conf needs to be set with appropriate class loader to resolve // quartz resources. conf.setClassLoader(QuartzConfiguration.class.getClassLoader()); return addQuartzResources(conf); }
From source file:com.asakusafw.lang.compiler.mapreduce.testing.MapReduceRunner.java
License:Apache License
private static int execute0(Configuration conf, ClassLoader classLoader, ClassDescription clientClass, String executionId, Map<String, String> batchArguments) throws Exception { try (ClassLoaderContext context = new ClassLoaderContext(classLoader)) { Configuration copy = new Configuration(conf); copy.setClassLoader(classLoader); Tool tool = resolveClient(copy, clientClass); configure(copy, executionId, batchArguments); return tool.run(EMPTY_ARGUMENTS); }//from w w w . j a v a2 s .co m }
From source file:com.asakusafw.m3bp.client.M3bpDirect.java
License:Apache License
/** * Program entry./*from www . j a v a2 s .c o m*/ * @param loader the launch class loader * @param args launching configurations * @return the exit code * @throws LaunchConfigurationException if launching configuration is something wrong * @see LaunchUtil#EXEC_SUCCESS * @see LaunchUtil#EXEC_ERROR * @see LaunchUtil#EXEC_INTERRUPTED */ public static int exec(ClassLoader loader, String... args) throws LaunchConfigurationException { DirectLaunchConfiguration conf = DirectLaunchConfiguration.parse(loader, Arrays.asList(args)); Configuration hadoop = new Configuration(); hadoop.setClassLoader(loader); int numberOfRounds = conf.getStageInfo().getRoundCount(); int currentRound = 0; DirectLaunchConfiguration.Cursor cursor = conf.newCursor(); while (cursor.next()) { LOG.info("Round: {}/{}", ++currentRound, numberOfRounds); int result = new M3bpLauncher(cursor.get(), hadoop).exec(); if (result != LaunchUtil.EXEC_SUCCESS) { return result; } } return LaunchUtil.EXEC_SUCCESS; }
From source file:com.asakusafw.runtime.stage.launcher.ApplicationLauncher.java
License:Apache License
/** * Executes launcher.//from w w w . j a v a 2 s .c o m * @param configuration the Hadoop configuration for the application * @param args the launcher arguments * @return the exit status */ public static int exec(Configuration configuration, String... args) { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Preparing application: {0}", //$NON-NLS-1$ Arrays.toString(args))); } configuration.setBoolean(KEY_LAUNCHER_USED, true); LauncherOptions options; try { options = LauncherOptionsParser.parse(configuration, args); } catch (Exception e) { LOG.error(MessageFormat.format("Exception occurred in launcher: {0}", Arrays.toString(args)), e); return LAUNCH_ERROR; } try { Configuration conf = options.getConfiguration(); conf.setClassLoader(options.getApplicationClassLoader()); Tool tool; try { tool = ReflectionUtils.newInstance(options.getApplicationClass(), conf); } catch (Exception e) { LOG.error(MessageFormat.format("Exception occurred in launcher: {0}{1}", options.getApplicationClass().getName(), options.getApplicationArguments()), e); return LAUNCH_ERROR; } try { return launch(conf, tool, options.getApplicationArgumentArray()); } catch (Exception e) { LOG.error(MessageFormat.format("Exception occurred in launcher: {0}{1}", options.getApplicationClass().getName(), options.getApplicationArguments()), e); return CLIENT_ERROR; } } finally { disposeClassLoader(options.getApplicationClassLoader()); for (File file : options.getApplicationCacheDirectories()) { if (delete(file) == false) { LOG.warn(MessageFormat.format("Failed to delete the application cache directory: {0}", file)); } } } }
From source file:com.asakusafw.runtime.util.hadoop.ConfigurationProvider.java
License:Apache License
/** * Creates a new default configuration./*w ww . j av a 2s . c o m*/ * @return the created configuration */ public Configuration newInstance() { Configuration conf = new Configuration(true); conf.setClassLoader(loader); configure(conf); return conf; }
From source file:com.asakusafw.testdriver.inprocess.InProcessHadoopTaskExecutor.java
License:Apache License
@Override public void execute(TaskExecutionContext context, TaskInfo task) throws InterruptedException, IOException { HadoopTaskInfo mirror = (HadoopTaskInfo) task; List<String> arguments = new ArrayList<>(); arguments.add(mirror.getClassName()); Map<String, String> properties = getHadoopProperties(context); TaskExecutors.withLibraries(context, classLoader -> { Configuration conf = context.findResource(Configuration.class).get(); TaskExecutors.findCoreConfigurationUrl(context).ifPresent(conf::addResource); properties.forEach(conf::set);//from w w w. j ava 2s . co m conf.setClassLoader(classLoader); LOG.info("starting Hadoop task: {} {}", mirror.getClassName()); int exit = ApplicationLauncher.exec(conf, arguments.toArray(new String[arguments.size()])); if (exit != 0) { throw new IOException(MessageFormat.format("failed to execute Hadoop task: class={0}, exit={1}", mirror.getClassName(), exit)); } }); }
From source file:com.asakusafw.vanilla.client.VanillaDirect.java
License:Apache License
/** * Program entry./* w w w. ja v a 2s. co m*/ * @param loader the launch class loader * @param args launching configurations * @return the exit code * @throws LaunchConfigurationException if launching configuration is something wrong * @see VanillaLauncher#EXEC_SUCCESS * @see VanillaLauncher#EXEC_ERROR * @see VanillaLauncher#EXEC_INTERRUPTED */ public static int exec(ClassLoader loader, String... args) throws LaunchConfigurationException { DirectLaunchConfiguration conf = DirectLaunchConfiguration.parse(loader, Arrays.asList(args)); Configuration hadoop = new Configuration(); hadoop.setClassLoader(loader); int numberOfRounds = conf.getStageInfo().getRoundCount(); int currentRound = 0; DirectLaunchConfiguration.Cursor cursor = conf.newCursor(); while (cursor.next()) { LOG.info("Round: {}/{}", ++currentRound, numberOfRounds); int result = new VanillaLauncher(cursor.get(), hadoop).exec(); if (result != VanillaLauncher.EXEC_SUCCESS) { return result; } } return VanillaLauncher.EXEC_SUCCESS; }