List of usage examples for org.apache.hadoop.conf Configuration getClassLoader
public ClassLoader getClassLoader()
From source file:co.cask.cdap.app.runtime.spark.SparkRuntimeUtils.java
License:Apache License
/** * Sets the context ClassLoader to the given {@link SparkClassLoader}. It will also set the * ClassLoader for the {@link Configuration} contained inside the {@link SparkClassLoader}. * * @return a {@link Cancellable} to reset the classloader to the one prior to the call *//* ww w. jav a 2s . c o m*/ public static Cancellable setContextClassLoader(final SparkClassLoader sparkClassLoader) { final Configuration hConf = sparkClassLoader.getRuntimeContext().getConfiguration(); final ClassLoader oldConfClassLoader = hConf.getClassLoader(); // Always wrap it with WeakReference to avoid ClassLoader leakage from Spark. ClassLoader classLoader = new WeakReferenceDelegatorClassLoader(sparkClassLoader); hConf.setClassLoader(classLoader); final ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(classLoader); return new Cancellable() { @Override public void cancel() { hConf.setClassLoader(oldConfClassLoader); ClassLoaders.setContextClassLoader(oldClassLoader); // Do not remove the next line. // This is necessary to keep a strong reference to the SparkClassLoader so that it won't get GC until this // cancel() is called LOG.trace("Reset context ClassLoader. The SparkClassLoader is: {}", sparkClassLoader); } }; }
From source file:co.cask.cdap.hive.datasets.DatasetAccessor.java
License:Apache License
public DatasetAccessor(Configuration conf) throws IOException { String datasetName = conf.get(Constants.Explore.DATASET_NAME); String namespace = conf.get(Constants.Explore.DATASET_NAMESPACE); Preconditions.checkArgument(!Strings.isNullOrEmpty(datasetName), "dataset name not present in config"); Preconditions.checkArgument(!Strings.isNullOrEmpty(namespace), "namespace not present in config"); this.datasetId = Id.DatasetInstance.from(namespace, datasetName); this.context = ContextManager.getContext(conf); this.transaction = ConfigurationUtil.get(conf, Constants.Explore.TX_QUERY_KEY, TxnCodec.INSTANCE); this.datasetInstantiator = context.createDatasetInstantiator(conf.getClassLoader()); }
From source file:co.cask.cdap.hive.datasets.DatasetSerDe.java
License:Apache License
private void getDatasetSchema(Configuration conf, Id.DatasetInstance datasetId) throws SerDeException { try (ContextManager.Context hiveContext = ContextManager.getContext(conf)) { // apparently the conf can be null in some versions of Hive? // Because it calls initialize just to get the object inspector if (hiveContext == null) { LOG.info("Hive provided a null conf, will not be able to get dataset schema."); return; }// w w w . ja v a 2 s .c o m // some datasets like Table and ObjectMappedTable have schema in the dataset properties try { DatasetSpecification datasetSpec = hiveContext.getDatasetSpec(datasetId); String schemaStr = datasetSpec.getProperty("schema"); if (schemaStr != null) { schema = Schema.parseJson(schemaStr); return; } } catch (DatasetManagementException | ServiceUnavailableException e) { throw new SerDeException("Could not instantiate dataset " + datasetId, e); } catch (IOException e) { throw new SerDeException("Exception getting schema for dataset " + datasetId, e); } // other datasets must be instantiated to get their schema // conf is null if this is a query that writes to a dataset ClassLoader parentClassLoader = conf == null ? null : conf.getClassLoader(); try (SystemDatasetInstantiator datasetInstantiator = hiveContext .createDatasetInstantiator(parentClassLoader)) { Dataset dataset = datasetInstantiator.getDataset(datasetId); if (dataset == null) { throw new SerDeException("Could not find dataset " + datasetId); } Type recordType; if (dataset instanceof RecordScannable) { recordType = ((RecordScannable) dataset).getRecordType(); } else if (dataset instanceof RecordWritable) { recordType = ((RecordWritable) dataset).getRecordType(); } else { throw new SerDeException("Dataset " + datasetId + " is not explorable."); } schema = schemaGenerator.generate(recordType); } catch (UnsupportedTypeException e) { throw new SerDeException("Dataset " + datasetId + " has an unsupported schema.", e); } catch (IOException e) { throw new SerDeException("Exception while trying to instantiate dataset " + datasetId, e); } } catch (IOException e) { throw new SerDeException("Could not get hive context from configuration.", e); } }
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceClassLoader.java
License:Apache License
/** * Finds the {@link MapReduceClassLoader} from the {@link ClassLoader} inside the given {@link Configuration}. * * @throws IllegalArgumentException if no {@link MapReduceClassLoader} can be found from the {@link Configuration}. *//*from w w w . j a v a 2s . c o m*/ public static MapReduceClassLoader getFromConfiguration(Configuration configuration) { return Delegators.getDelegate(configuration.getClassLoader(), MapReduceClassLoader.class); }
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceContextProvider.java
License:Apache License
/** * Returns the {@link ClassLoader} for the MapReduce program. The ClassLoader for MapReduce job is always * an {@link MapReduceClassLoader}, which set by {@link MapReduceRuntimeService} in local mode and created by MR * framework in distributed mode.//w w w . ja va2s. com */ static ClassLoader getProgramClassLoader(Configuration hConf) { ClassLoader classLoader = Delegators.getDelegate(hConf.getClassLoader(), MapReduceClassLoader.class); if (!(classLoader instanceof MapReduceClassLoader)) { throw new IllegalArgumentException("ClassLoader is not an MapReduceClassLoader"); } return ((MapReduceClassLoader) classLoader).getProgramClassLoader(); }
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.java
License:Apache License
@Override protected void startUp() throws Exception { // Creates a temporary directory locally for storing all generated files. File tempDir = createTempDirectory(); cleanupTask = createCleanupTask(tempDir); try {/* w ww.j a v a2s .c om*/ Job job = createJob(new File(tempDir, "mapreduce")); Configuration mapredConf = job.getConfiguration(); classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getPlugins(), context.getPluginInstantiator()); cleanupTask = createCleanupTask(cleanupTask, classLoader); mapredConf.setClassLoader(new WeakReferenceDelegatorClassLoader(classLoader)); ClassLoaders.setContextClassLoader(mapredConf.getClassLoader()); context.setJob(job); beforeSubmit(job); // Localize additional resources that users have requested via BasicMapReduceContext.localize methods Map<String, String> localizedUserResources = localizeUserResources(job, tempDir); // Override user-defined job name, since we set it and depend on the name. // https://issues.cask.co/browse/CDAP-2441 String jobName = job.getJobName(); if (!jobName.isEmpty()) { LOG.warn("Job name {} is being overridden.", jobName); } job.setJobName(getJobName(context)); // Create a temporary location for storing all generated files through the LocationFactory. Location tempLocation = createTempLocationDirectory(); cleanupTask = createCleanupTask(cleanupTask, tempLocation); // For local mode, everything is in the configuration classloader already, hence no need to create new jar if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // After calling beforeSubmit, we know what plugins are needed for the program, hence construct the proper // ClassLoader from here and use it for setting up the job Location pluginArchive = createPluginArchive(tempLocation); if (pluginArchive != null) { job.addCacheArchive(pluginArchive.toURI()); mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName()); } } // set resources for the job TaskType.MAP.setResources(mapredConf, context.getMapperResources()); TaskType.REDUCE.setResources(mapredConf, context.getReducerResources()); // replace user's Mapper & Reducer's with our wrappers in job config MapperWrapper.wrap(job); ReducerWrapper.wrap(job); // packaging job jar which includes cdap classes with dependencies File jobJar = buildJobJar(job, tempDir); job.setJar(jobJar.toURI().toString()); Location programJar = programJarLocation; if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // Copy and localize the program jar in distributed mode programJar = copyProgramJar(tempLocation); job.addCacheFile(programJar.toURI()); List<String> classpath = new ArrayList<>(); // Localize logback.xml Location logbackLocation = createLogbackJar(tempLocation); if (logbackLocation != null) { job.addCacheFile(logbackLocation.toURI()); classpath.add(logbackLocation.getName()); } // Generate and localize the launcher jar to control the classloader of MapReduce containers processes classpath.add("job.jar/lib/*"); classpath.add("job.jar/classes"); Location launcherJar = createLauncherJar( Joiner.on(",").join(MapReduceContainerHelper.getMapReduceClassPath(mapredConf, classpath)), tempLocation); job.addCacheFile(launcherJar.toURI()); // The only thing in the container classpath is the launcher.jar // The MapReduceContainerLauncher inside the launcher.jar will creates a MapReduceClassLoader and launch // the actual MapReduce AM/Task from that // We explicitly localize the mr-framwork, but not use it with the classpath URI frameworkURI = MapReduceContainerHelper.getFrameworkURI(mapredConf); if (frameworkURI != null) { job.addCacheArchive(frameworkURI); } mapredConf.unset(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, launcherJar.getName()); mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, launcherJar.getName()); } MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); // We start long-running tx to be used by mapreduce job tasks. Transaction tx = txClient.startLong(); try { // We remember tx, so that we can re-use it in mapreduce tasks CConfiguration cConfCopy = cConf; contextConfig.set(context, cConfCopy, tx, programJar.toURI(), localizedUserResources); LOG.info("Submitting MapReduce Job: {}", context); // submits job and returns immediately. Shouldn't need to set context ClassLoader. job.submit(); this.job = job; this.transaction = tx; } catch (Throwable t) { Transactions.invalidateQuietly(txClient, tx); throw t; } } catch (Throwable t) { LOG.error("Exception when submitting MapReduce Job: {}", context, t); cleanupTask.run(); throw t; } }
From source file:co.cask.cdap.template.etl.common.ETLDBInputFormat.java
License:Apache License
@Override public Connection getConnection() { if (this.connection == null) { Configuration conf = getConf(); try {//from ww w. java 2s . c o m String url = conf.get(DBConfiguration.URL_PROPERTY); try { // throws SQLException if no suitable driver is found DriverManager.getDriver(url); } catch (SQLException e) { if (driverShim == null) { if (driver == null) { ClassLoader classLoader = conf.getClassLoader(); @SuppressWarnings("unchecked") Class<? extends Driver> driverClass = (Class<? extends Driver>) classLoader .loadClass(conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY)); driver = driverClass.newInstance(); // De-register the default driver that gets registered when driver class is loaded. DBUtils.deregisterAllDrivers(driverClass); } driverShim = new JDBCDriverShim(driver); DriverManager.registerDriver(driverShim); LOG.debug("Registered JDBC driver via shim {}. Actual Driver {}.", driverShim, driver); } } if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) { this.connection = DriverManager.getConnection(url); } else { this.connection = DriverManager.getConnection(url, conf.get(DBConfiguration.USERNAME_PROPERTY), conf.get(DBConfiguration.PASSWORD_PROPERTY)); } this.connection.setAutoCommit(false); this.connection.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); } catch (Exception e) { throw Throwables.propagate(e); } } return this.connection; }
From source file:co.cask.cdap.template.etl.common.ETLDBOutputFormat.java
License:Apache License
private Connection getConnection(Configuration conf) { Connection connection;//from w w w . java 2 s. c o m try { String url = conf.get(DBConfiguration.URL_PROPERTY); try { // throws SQLException if no suitable driver is found DriverManager.getDriver(url); } catch (SQLException e) { if (driverShim == null) { if (driver == null) { ClassLoader classLoader = conf.getClassLoader(); @SuppressWarnings("unchecked") Class<? extends Driver> driverClass = (Class<? extends Driver>) classLoader .loadClass(conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY)); driver = driverClass.newInstance(); // De-register the default driver that gets registered when driver class is loaded. DBUtils.deregisterAllDrivers(driverClass); } driverShim = new JDBCDriverShim(driver); DriverManager.registerDriver(driverShim); LOG.debug("Registered JDBC driver via shim {}. Actual Driver {}.", driverShim, driver); } } if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) { connection = DriverManager.getConnection(url); } else { connection = DriverManager.getConnection(url, conf.get(DBConfiguration.USERNAME_PROPERTY), conf.get(DBConfiguration.PASSWORD_PROPERTY)); } connection.setAutoCommit(false); connection.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); } catch (Exception e) { throw Throwables.propagate(e); } return connection; }
From source file:co.cask.hydrator.plugin.db.batch.sink.ETLDBOutputFormat.java
License:Apache License
private Connection getConnection(Configuration conf) { Connection connection;//w w w . ja va 2 s. c o m try { String url = conf.get(DBConfiguration.URL_PROPERTY); try { // throws SQLException if no suitable driver is found DriverManager.getDriver(url); } catch (SQLException e) { if (driverShim == null) { if (driver == null) { ClassLoader classLoader = conf.getClassLoader(); @SuppressWarnings("unchecked") Class<? extends Driver> driverClass = (Class<? extends Driver>) classLoader .loadClass(conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY)); driver = driverClass.newInstance(); // De-register the default driver that gets registered when driver class is loaded. DBUtils.deregisterAllDrivers(driverClass); } driverShim = new JDBCDriverShim(driver); DriverManager.registerDriver(driverShim); LOG.debug("Registered JDBC driver via shim {}. Actual Driver {}.", driverShim, driver); } } if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) { connection = DriverManager.getConnection(url); } else { connection = DriverManager.getConnection(url, conf.get(DBConfiguration.USERNAME_PROPERTY), conf.get(DBConfiguration.PASSWORD_PROPERTY)); } boolean autoCommitEnabled = conf.getBoolean(AUTO_COMMIT_ENABLED, false); if (autoCommitEnabled) { // hack to work around jdbc drivers like the hive driver that throw exceptions on commit connection = new NoOpCommitConnection(connection); } else { connection.setAutoCommit(false); } connection.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); } catch (Exception e) { throw Throwables.propagate(e); } return connection; }
From source file:co.cask.hydrator.plugin.db.batch.source.DataDrivenETLDBInputFormat.java
License:Apache License
@Override public Connection getConnection() { if (this.connection == null) { Configuration conf = getConf(); try {//ww w. j a va 2s.c o m String url = conf.get(DBConfiguration.URL_PROPERTY); try { // throws SQLException if no suitable driver is found DriverManager.getDriver(url); } catch (SQLException e) { if (driverShim == null) { if (driver == null) { ClassLoader classLoader = conf.getClassLoader(); @SuppressWarnings("unchecked") Class<? extends Driver> driverClass = (Class<? extends Driver>) classLoader .loadClass(conf.get(DBConfiguration.DRIVER_CLASS_PROPERTY)); driver = driverClass.newInstance(); // De-register the default driver that gets registered when driver class is loaded. DBUtils.deregisterAllDrivers(driverClass); } driverShim = new JDBCDriverShim(driver); DriverManager.registerDriver(driverShim); LOG.debug("Registered JDBC driver via shim {}. Actual Driver {}.", driverShim, driver); } } if (conf.get(DBConfiguration.USERNAME_PROPERTY) == null) { this.connection = DriverManager.getConnection(url); } else { this.connection = DriverManager.getConnection(url, conf.get(DBConfiguration.USERNAME_PROPERTY), conf.get(DBConfiguration.PASSWORD_PROPERTY)); } boolean autoCommitEnabled = conf.getBoolean(AUTO_COMMIT_ENABLED, false); if (autoCommitEnabled) { // hack to work around jdbc drivers like the hive driver that throw exceptions on commit this.connection = new NoOpCommitConnection(this.connection); } else { this.connection.setAutoCommit(false); } this.connection.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); } catch (Exception e) { throw Throwables.propagate(e); } } return this.connection; }