List of usage examples for org.apache.hadoop.conf Configuration unset
public synchronized void unset(String name)
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.java
License:Apache License
@Override protected void startUp() throws Exception { // Creates a temporary directory locally for storing all generated files. File tempDir = createTempDirectory(); cleanupTask = createCleanupTask(tempDir); try {//from w w w . j ava2s . c o m Job job = createJob(new File(tempDir, "mapreduce")); Configuration mapredConf = job.getConfiguration(); classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getPlugins(), context.getPluginInstantiator()); cleanupTask = createCleanupTask(cleanupTask, classLoader); mapredConf.setClassLoader(new WeakReferenceDelegatorClassLoader(classLoader)); ClassLoaders.setContextClassLoader(mapredConf.getClassLoader()); context.setJob(job); beforeSubmit(job); // Localize additional resources that users have requested via BasicMapReduceContext.localize methods Map<String, String> localizedUserResources = localizeUserResources(job, tempDir); // Override user-defined job name, since we set it and depend on the name. // https://issues.cask.co/browse/CDAP-2441 String jobName = job.getJobName(); if (!jobName.isEmpty()) { LOG.warn("Job name {} is being overridden.", jobName); } job.setJobName(getJobName(context)); // Create a temporary location for storing all generated files through the LocationFactory. Location tempLocation = createTempLocationDirectory(); cleanupTask = createCleanupTask(cleanupTask, tempLocation); // For local mode, everything is in the configuration classloader already, hence no need to create new jar if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // After calling beforeSubmit, we know what plugins are needed for the program, hence construct the proper // ClassLoader from here and use it for setting up the job Location pluginArchive = createPluginArchive(tempLocation); if (pluginArchive != null) { job.addCacheArchive(pluginArchive.toURI()); mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName()); } } // set resources for the job TaskType.MAP.setResources(mapredConf, context.getMapperResources()); TaskType.REDUCE.setResources(mapredConf, context.getReducerResources()); // replace user's Mapper & Reducer's with our wrappers in job config MapperWrapper.wrap(job); ReducerWrapper.wrap(job); // packaging job jar which includes cdap classes with dependencies File jobJar = buildJobJar(job, tempDir); job.setJar(jobJar.toURI().toString()); Location programJar = programJarLocation; if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // Copy and localize the program jar in distributed mode programJar = copyProgramJar(tempLocation); job.addCacheFile(programJar.toURI()); List<String> classpath = new ArrayList<>(); // Localize logback.xml Location logbackLocation = createLogbackJar(tempLocation); if (logbackLocation != null) { job.addCacheFile(logbackLocation.toURI()); classpath.add(logbackLocation.getName()); } // Generate and localize the launcher jar to control the classloader of MapReduce containers processes classpath.add("job.jar/lib/*"); classpath.add("job.jar/classes"); Location launcherJar = createLauncherJar( Joiner.on(",").join(MapReduceContainerHelper.getMapReduceClassPath(mapredConf, classpath)), tempLocation); job.addCacheFile(launcherJar.toURI()); // The only thing in the container classpath is the launcher.jar // The MapReduceContainerLauncher inside the launcher.jar will creates a MapReduceClassLoader and launch // the actual MapReduce AM/Task from that // We explicitly localize the mr-framwork, but not use it with the classpath URI frameworkURI = MapReduceContainerHelper.getFrameworkURI(mapredConf); if (frameworkURI != null) { job.addCacheArchive(frameworkURI); } mapredConf.unset(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, launcherJar.getName()); mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, launcherJar.getName()); } MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); // We start long-running tx to be used by mapreduce job tasks. Transaction tx = txClient.startLong(); try { // We remember tx, so that we can re-use it in mapreduce tasks CConfiguration cConfCopy = cConf; contextConfig.set(context, cConfCopy, tx, programJar.toURI(), localizedUserResources); LOG.info("Submitting MapReduce Job: {}", context); // submits job and returns immediately. Shouldn't need to set context ClassLoader. job.submit(); this.job = job; this.transaction = tx; } catch (Throwable t) { Transactions.invalidateQuietly(txClient, tx); throw t; } } catch (Throwable t) { LOG.error("Exception when submitting MapReduce Job: {}", context, t); cleanupTask.run(); throw t; } }
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.java
License:Apache License
/** * Creates a MapReduce {@link Job} instance. * * @param hadoopTmpDir directory for the "hadoop.tmp.dir" configuration *//*from w w w . jav a2s . c o m*/ private Job createJob(File hadoopTmpDir) throws IOException { Job job = Job.getInstance(new Configuration(hConf)); Configuration jobConf = job.getConfiguration(); if (MapReduceTaskContextProvider.isLocal(jobConf)) { // Set the MR framework local directories inside the given tmp directory. // Setting "hadoop.tmp.dir" here has no effect due to Explore Service need to set "hadoop.tmp.dir" // as system property for Hive to work in local mode. The variable substitution of hadoop conf // gives system property the highest precedence. jobConf.set("mapreduce.cluster.local.dir", new File(hadoopTmpDir, "local").getAbsolutePath()); jobConf.set("mapreduce.jobtracker.system.dir", new File(hadoopTmpDir, "system").getAbsolutePath()); jobConf.set("mapreduce.jobtracker.staging.root.dir", new File(hadoopTmpDir, "staging").getAbsolutePath()); jobConf.set("mapreduce.cluster.temp.dir", new File(hadoopTmpDir, "temp").getAbsolutePath()); } if (UserGroupInformation.isSecurityEnabled()) { // If runs in secure cluster, this program runner is running in a yarn container, hence not able // to get authenticated with the history. jobConf.unset("mapreduce.jobhistory.address"); jobConf.setBoolean(Job.JOB_AM_ACCESS_DISABLED, false); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Running in secure mode; adding all user credentials: {}", credentials.getAllTokens()); job.getCredentials().addAll(credentials); } return job; }
From source file:co.cask.hydrator.common.batch.JobUtils.java
License:Apache License
/** * Creates a new instance of {@link Job}. Note that the job created is not meant for actual MR * submission. It's just for setting up configurations. *//*from w w w. j a v a 2 s . co m*/ public static Job createInstance() throws IOException { Job job = Job.getInstance(); Configuration conf = job.getConfiguration(); conf.clear(); if (UserGroupInformation.isSecurityEnabled()) { // If runs in secure cluster, this program runner is running in a yarn container, hence not able // to get authenticated with the history. conf.unset("mapreduce.jobhistory.address"); conf.setBoolean(Job.JOB_AM_ACCESS_DISABLED, false); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); job.getCredentials().addAll(credentials); } return job; }
From source file:co.cask.tephra.coprocessor.TransactionStateCache.java
License:Apache License
protected Configuration getSnapshotConfiguration() throws IOException { Configuration conf = new ConfigurationFactory().get(hConf); conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES); return conf;/*from w ww . j a v a 2 s . c o m*/ }
From source file:co.cask.tephra.hbase.AbstractTransactionVisibilityFilterTest.java
License:Apache License
@Before public void setup() throws Exception { Configuration conf = new ConfigurationFactory().get(); conf.unset(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES); txManager = new TransactionManager(conf); txManager.startAndWait();/* w w w. java 2s . co m*/ }
From source file:com.aliyun.openservices.tablestore.hadoop.TableStoreInputFormat.java
License:Apache License
/** * Clear TableStore data sources.//from w w w .j a va 2 s . c om */ public static void clearCriteria(Configuration conf) { Preconditions.checkNotNull(conf, "conf must be nonnull"); conf.unset(CRITERIA); }
From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java
License:Apache License
@Test public void testGetNodes() throws Exception { try {//from ww w . jav a2 s .c o m startYarn(createMiniYarnConfig(true)); //we have to wait a bit to ensure the RM got the NM registered Thread.sleep(3000); Configuration conf = getLlamaConfiguration(); conf.unset(LlamaAM.CORE_QUEUES_KEY); LlamaAM llama = LlamaAM.create(conf); try { llama.start(); List<NodeInfo> nodes = llama.getNodes(); Assert.assertFalse(nodes.isEmpty()); } finally { llama.stop(); } } finally { stopYarn(); } }
From source file:com.cloudera.recordservice.mapreduce.MapReduceTest.java
License:Apache License
@Test public void testGetSplits() throws IOException { Configuration config = new Configuration(); boolean exceptionThrown = false; try {/* w w w .j ava 2 s .c om*/ PlanUtil.getSplits(config, new Credentials()); } catch (IllegalArgumentException e) { exceptionThrown = true; assertTrue(e.getMessage().contains("No input specified")); } assertTrue(exceptionThrown); // Set db/table and make sure it works. config.set(ConfVars.TBL_NAME_CONF.name, "tpch.nation"); PlanUtil.getSplits(config, new Credentials()); // Also set input. This should fail. config.set(FileInputFormat.INPUT_DIR, "/test"); exceptionThrown = false; try { PlanUtil.getSplits(config, new Credentials()); } catch (IllegalArgumentException e) { exceptionThrown = true; assertTrue(e.getMessage(), e.getMessage().contains("More than one input specified")); } assertTrue(exceptionThrown); // Unset the table and set columns. INPUT_DIR and columns don't work now. config.unset(ConfVars.TBL_NAME_CONF.name); config.setStrings(ConfVars.COL_NAMES_CONF.name, "a"); exceptionThrown = false; try { PlanUtil.getSplits(config, new Credentials()); } catch (IllegalArgumentException e) { exceptionThrown = true; assertTrue(e.getMessage().contains("Column projections can only be specified with table inputs.")); } assertTrue(exceptionThrown); // Test some cases that work verifyInputSplitsTable(1, 4, "tpch.nation"); verifyInputSplitsTable(2, 12, "rs.alltypes"); verifyInputSplitsTable(1, 1, "tpch.nation", "n_name"); verifyInputSplitsTable(2, 3, "rs.alltypes", "int_col", "double_col", "string_col"); verifyInputSplitsPath(1, 1, "/test-warehouse/tpch.nation"); // Test some cases using the config utility. config.clear(); RecordServiceConfig.setInputTable(config, null, "tpch.nation", "n_nationkey", "n_comment"); verifyInputSplits(1, 2, config); exceptionThrown = false; try { verifyInputSplitsTable(1, 1, "tpch.nation", "bad"); } catch (IOException e) { exceptionThrown = true; assertTrue(e.getCause() instanceof RecordServiceException); RecordServiceException ex = (RecordServiceException) e.getCause(); assertEquals(RecordServiceException.ErrorCode.INVALID_REQUEST, ex.code); } assertTrue(exceptionThrown); exceptionThrown = false; try { verifyInputSplitsPath(1, 1, "/test-warehouse/tpch.nation,/test-warehouse/tpch.nation"); } catch (IllegalArgumentException e) { exceptionThrown = true; assertTrue(e.getMessage().contains("Only reading a single directory is currently supported.")); } assertTrue(exceptionThrown); }
From source file:com.facebook.presto.hive.HdfsConfigurationUpdater.java
License:Apache License
public static void configureCompression(Configuration config, HiveCompressionCodec compressionCodec) { boolean compression = compressionCodec != HiveCompressionCodec.NONE; config.setBoolean(COMPRESSRESULT.varname, compression); config.setBoolean("mapred.output.compress", compression); config.setBoolean(FileOutputFormat.COMPRESS, compression); // For DWRF//from www . java 2s. com config.set(HIVE_ORC_DEFAULT_COMPRESS.varname, compressionCodec.getOrcCompressionKind().name()); config.set(HIVE_ORC_COMPRESSION.varname, compressionCodec.getOrcCompressionKind().name()); // For ORC config.set(OrcTableProperties.COMPRESSION.getPropName(), compressionCodec.getOrcCompressionKind().name()); // For RCFile and Text if (compressionCodec.getCodec().isPresent()) { config.set("mapred.output.compression.codec", compressionCodec.getCodec().get().getName()); config.set(FileOutputFormat.COMPRESS_CODEC, compressionCodec.getCodec().get().getName()); } else { config.unset("mapred.output.compression.codec"); config.unset(FileOutputFormat.COMPRESS_CODEC); } // For Parquet config.set(ParquetOutputFormat.COMPRESSION, compressionCodec.getParquetCompressionCodec().name()); // For SequenceFile config.set(FileOutputFormat.COMPRESS_TYPE, BLOCK.toString()); }
From source file:com.google.cloud.bigtable.hbase.TestBigtableOptionsFactory.java
License:Open Source License
@Test public void testProjectIdIsRequired() throws IOException { Configuration configuration = new Configuration(false); configuration.unset(BigtableOptionsFactory.PROJECT_ID_KEY); expectedException.expect(IllegalArgumentException.class); BigtableOptionsFactory.fromConfiguration(configuration); }