List of usage examples for org.apache.hadoop.mapreduce MRJobConfig DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH
String DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH
To view the source code for org.apache.hadoop.mapreduce MRJobConfig DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH.
Click Source Link
From source file:co.cask.cdap.explore.service.ExploreServiceUtils.java
License:Apache License
/** * Change mapred-site.xml file, and return a temp copy of it to which are added * necessary options.//from www .ja va 2 s. co m */ private static File updateMapredConfFile(File confFile, File tempDir) { Configuration conf = new Configuration(false); try { conf.addResource(confFile.toURI().toURL()); } catch (MalformedURLException e) { LOG.error("File {} is malformed.", confFile, e); throw Throwables.propagate(e); } String mrAppClassPath = conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH); // Add the pwd/* at the beginning of classpath. Without this change, old jars from mr framework classpath // get into classpath. mrAppClassPath = "$PWD/*," + mrAppClassPath; conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, mrAppClassPath); File newMapredConfFile = new File(tempDir, "mapred-site.xml"); try (FileOutputStream os = new FileOutputStream(newMapredConfFile)) { conf.writeXml(os); } catch (IOException e) { LOG.error("Problem creating and writing to temporary mapred-site.xml conf file at {}", newMapredConfFile, e); throw Throwables.propagate(e); } return newMapredConfFile; }
From source file:co.cask.cdap.explore.service.ExploreServiceUtilsTest.java
License:Apache License
@Test public void hijackConfFileTest() throws Exception { Configuration conf = new Configuration(false); conf.set("foo", "bar"); Assert.assertEquals(1, conf.size()); File tempDir = tmpFolder.newFolder(); File confFile = tmpFolder.newFile("hive-site.xml"); try (FileOutputStream os = new FileOutputStream(confFile)) { conf.writeXml(os);//from w ww .jav a2 s . com } File newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir); conf = new Configuration(false); conf.addResource(newConfFile.toURI().toURL()); Assert.assertEquals(3, conf.size()); Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_USER_CLASSPATH_FIRST)); Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_CLASSLOADER)); Assert.assertEquals("bar", conf.get("foo")); // check yarn-site changes confFile = tmpFolder.newFile("yarn-site.xml"); conf = new YarnConfiguration(); try (FileOutputStream os = new FileOutputStream(confFile)) { conf.writeXml(os); } String yarnApplicationClassPath = "$PWD/*," + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir); conf = new Configuration(false); conf.addResource(newConfFile.toURI().toURL()); Assert.assertEquals(yarnApplicationClassPath, conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH)); // check mapred-site changes confFile = tmpFolder.newFile("mapred-site.xml"); conf = new YarnConfiguration(); try (FileOutputStream os = new FileOutputStream(confFile)) { conf.writeXml(os); } String mapredApplicationClassPath = "$PWD/*," + conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH); newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir); conf = new Configuration(false); conf.addResource(newConfFile.toURI().toURL()); Assert.assertEquals(mapredApplicationClassPath, conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH)); // Ensure conf files that are not hive-site.xml/mapred-site.xml/yarn-site.xml are unchanged confFile = tmpFolder.newFile("core-site.xml"); Assert.assertEquals(confFile, ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir)); }
From source file:co.cask.cdap.internal.app.runtime.batch.distributed.MapReduceContainerHelper.java
License:Apache License
/** * Returns a list of path to be used for the MapReduce framework classpath. * * @param hConf the configuration for the job. * @param result a list for appending MR framework classpath * @return the same {@code result} list from the argument *///w ww. j a v a 2s . c o m public static List<String> getMapReduceClassPath(Configuration hConf, List<String> result) { String framework = hConf.get(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); // For classpath config get from the hConf, we splits it with both "," and ":" because one can set // the conf with something like "path1,path2:path3" and // it should become "path1:path2:path3" in the target JVM process Splitter splitter = Splitter.on(Pattern.compile(",|" + File.pathSeparatorChar)).trimResults() .omitEmptyStrings(); // If MR framework is non specified, use yarn.application.classpath and mapreduce.application.classpath // Otherwise, only use the mapreduce.application.classpath if (framework == null) { String yarnClassPath = hConf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); Iterables.addAll(result, splitter.split(yarnClassPath)); } // Add MR application classpath Iterables.addAll(result, splitter.split(hConf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))); return result; }
From source file:ml.shifu.guagua.yarn.util.YarnUtils.java
License:Apache License
/** * Populate the environment string map to be added to the environment vars in a remote execution container. * //from www . ja va2s . c o m * @param env * the map of env var values. * @param conf * the Configuration to pull values from. */ public static void addLocalClasspathToEnv(final Map<String, String> env, final Configuration conf) { StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()); // add current folder classPathEnv.append(File.pathSeparatorChar).append("./*"); // add yarn app classpath for (String cpEntry : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar).append(cpEntry.trim()); } for (String jar : conf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, org.apache.hadoop.util.StringUtils .getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) { classPathEnv.append(File.pathSeparatorChar).append(jar.trim()); } // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(File.pathSeparatorChar).append(Environment.CLASSPATH.$()); } // add guagua app jar file String path = getFileName(conf.get(GuaguaYarnConstants.GUAGUA_YARN_APP_JAR)); classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(path).append(File.pathSeparatorChar); // Any libraries we may have copied over? String libs = conf.get(GuaguaYarnConstants.GUAGUA_YARN_APP_LIB_JAR); if (StringUtils.isNotEmpty(libs)) { for (String jar : Splitter.on(GuaguaYarnConstants.GUAGUA_APP_LIBS_SEPERATOR).split(libs)) { classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(getFileName(jar.trim())) .append(File.pathSeparatorChar); } } // add log4j classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(GuaguaYarnConstants.GUAGUA_LOG4J_PROPERTIES) .append(File.pathSeparatorChar); // add guagua-conf.xml classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(GuaguaYarnConstants.GUAGUA_CONF_FILE); env.put(Environment.CLASSPATH.toString(), classPathEnv.toString()); }
From source file:org.apache.giraph.yarn.YarnUtils.java
License:Apache License
/** * Popuate the environment string map to be added to the environment vars * in a remote execution container. Adds the local classpath to pick up * "yarn-site.xml" and "mapred-site.xml" stuff. * @param env the map of env var values. * @param giraphConf the GiraphConfiguration to pull values from. *//*from w ww.jav a2s .c om*/ public static void addLocalClasspathToEnv(final Map<String, String> env, final GiraphConfiguration giraphConf) { StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*"); for (String cpEntry : giraphConf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(':').append(cpEntry.trim()); //TODO: Separator } for (String cpEntry : giraphConf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) { classPathEnv.append(':').append(cpEntry.trim()); } // add the runtime classpath needed for tests to work if (giraphConf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':').append(System.getenv("CLASSPATH")); } env.put("CLASSPATH", classPathEnv.toString()); }
From source file:org.apache.oozie.util.ClasspathUtils.java
License:Apache License
public static void addMapReduceToClasspath(Map<String, String> env, Configuration conf) { boolean crossPlatform = conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM); for (String c : conf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, crossPlatform/* w w w. j ava 2 s . c om*/ ? StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH) : StringUtils.getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) { MRApps.addToEnvironment(env, ApplicationConstants.Environment.CLASSPATH.name(), c.trim(), conf); } }