List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APPLICATION_CLASSPATH
String YARN_APPLICATION_CLASSPATH
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APPLICATION_CLASSPATH.
Click Source Link
From source file:alluxio.yarn.Client.java
License:Apache License
private void setupAppMasterEnv(Map<String, String> appMasterEnv) throws IOException { String classpath = ApplicationConstants.Environment.CLASSPATH.name(); for (String path : mYarnConf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { Apps.addToEnvironment(appMasterEnv, classpath, path.trim(), ApplicationConstants.CLASS_PATH_SEPARATOR); }/*from w ww . j av a 2 s .c o m*/ Apps.addToEnvironment(appMasterEnv, classpath, PathUtils.concatPath(Environment.PWD.$(), "*"), ApplicationConstants.CLASS_PATH_SEPARATOR); appMasterEnv.put("ALLUXIO_HOME", ApplicationConstants.Environment.PWD.$()); if (UserGroupInformation.isSecurityEnabled()) { appMasterEnv.put("ALLUXIO_USER", UserGroupInformation.getCurrentUser().getShortUserName()); } }
From source file:base.test.HelloWorld.java
License:Apache License
public static void main(String[] args) { if (args.length < 1) { System.err.println("Arguments format: <host:port of zookeeper server>"); System.exit(1);//from w w w .j a v a 2 s . com } String zkStr = args[0]; YarnConfiguration yarnConfiguration = new YarnConfiguration(); yarnConfiguration.setSocketAddr("yarn.resourcemanager.address", new InetSocketAddress("192.168.80.103", 8032)); final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr); twillRunner.start(); String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); List<String> applicationClassPaths = Lists.newArrayList(); Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath)); final TwillController controller = twillRunner.prepare(new HelloWorldRunnable()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationClassPaths(applicationClassPaths) .withBundlerClassAcceptor(new HadoopClassExcluder()).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { controller.awaitTerminated(); } catch (ExecutionException e) { e.printStackTrace(); } }
From source file:cn.edu.buaa.act.petuumOnYarn.Client.java
License:Apache License
/** * Main run function for the client// w w w .j a v a 2 s. co m * * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); String[] s; s = conf.getStrings(YarnConfiguration.RM_ADDRESS); for (String ss : s) LOG.info("RM address: " + ss); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers() + ", nodeIdHost" + node.getNodeId().getHost()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of // the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); YarnUtil.copyAndAddToLocalResources(fs, appMasterJar, petuumHDFSPathPrefix, appMasterJarPath, localResources, null); scriptHDFSPath = YarnUtil.copyToHDFS(fs, scriptPath, petuumHDFSPathPrefix, launchPath, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { YarnUtil.copyAndAddToLocalResources(fs, log4jPropFile, petuumHDFSPathPrefix, log4jPath, localResources, null); } // Set the env variables to be setup in the env where the application // master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_nodes " + String.valueOf(numNodes)); vargs.add("--start_port " + String.valueOf(startPort)); vargs.add("--priority " + String.valueOf(workerPriority)); vargs.add("--script_hdfs_path " + scriptHDFSPath); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and // MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = // applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on // success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application currentTime = System.currentTimeMillis(); LOG.info("submit AM in " + (currentTime - startTime) + "ms"); return monitorApplication(appId); }
From source file:co.cask.cdap.data.runtime.main.MasterServiceMain.java
License:Apache License
/** * Starts the {@link TwillApplication} for the master services. * * @return The {@link TwillController} for the application. *///from w w w . ja va 2 s. co m private TwillController startTwillApplication(TwillRunnerService twillRunner) { try { // Create a temp dir for the run to hold temporary files created to run the application Path tempPath = Files.createDirectories( new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)) .toPath()); final Path runDir = Files.createTempDirectory(tempPath, "master"); try { Path cConfFile = saveCConf(cConf, runDir.resolve("cConf.xml")); Path hConfFile = saveHConf(hConf, runDir.resolve("hConf.xml")); Path logbackFile = saveLogbackConf(runDir.resolve("logback.xml")); TwillPreparer preparer = twillRunner.prepare(new MasterTwillApplication(cConf, cConfFile.toFile(), hConfFile.toFile(), getSystemServiceInstances())); if (cConf.getBoolean(Constants.COLLECT_CONTAINER_LOGS)) { if (LOG instanceof ch.qos.logback.classic.Logger) { preparer.addLogHandler(new LogHandler() { @Override public void onLog(LogEntry entry) { ch.qos.logback.classic.Logger logger = (ch.qos.logback.classic.Logger) LOG; logger.callAppenders(new TwillLogEntryAdapter(entry)); } }); } else { LOG.warn( "Unsupported logger binding ({}) for container log collection. Falling back to System.out.", LOG.getClass().getName()); preparer.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))); } } else { preparer.addJVMOptions("-Dtwill.disable.kafka=true"); } // Add logback xml if (Files.exists(logbackFile)) { preparer.withResources().withResources(logbackFile.toUri()); } // Add yarn queue name if defined String queueName = cConf.get(Constants.Service.SCHEDULER_QUEUE); if (queueName != null) { LOG.info("Setting scheduler queue to {} for master services", queueName); preparer.setSchedulerQueue(queueName); } // Add HBase dependencies preparer.withDependencies(baseInjector.getInstance(HBaseTableUtil.class).getClass()); // Add secure tokens if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) { // TokenSecureStoreUpdater.update() ignores parameters preparer.addSecureStore(secureStoreUpdater.update(null, null)); } // add hadoop classpath to application classpath and exclude hadoop classes from bundle jar. String yarnAppClassPath = hConf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); preparer.withApplicationClassPaths(Splitter.on(",").trimResults().split(yarnAppClassPath)) .withBundlerClassAcceptor(new HadoopClassExcluder()); // Add explore dependencies if (cConf.getBoolean(Constants.Explore.EXPLORE_ENABLED)) { prepareExploreContainer(preparer); } // Add a listener to delete temp files when application started/terminated. TwillController controller = preparer.start(); Runnable cleanup = new Runnable() { @Override public void run() { try { File dir = runDir.toFile(); if (dir.isDirectory()) { DirUtils.deleteDirectoryContents(dir); } } catch (IOException e) { LOG.warn("Failed to cleanup directory {}", runDir, e); } } }; controller.onRunning(cleanup, Threads.SAME_THREAD_EXECUTOR); controller.onTerminated(cleanup, Threads.SAME_THREAD_EXECUTOR); return controller; } catch (Exception e) { try { DirUtils.deleteDirectoryContents(runDir.toFile()); } catch (IOException ex) { LOG.warn("Failed to cleanup directory {}", runDir, ex); e.addSuppressed(ex); } throw e; } } catch (IOException e) { throw Throwables.propagate(e); } }
From source file:co.cask.cdap.explore.service.ExploreServiceUtils.java
License:Apache License
/** * Change yarn-site.xml file, and return a temp copy of it to which are added * necessary options./* ww w . ja va2s .c o m*/ */ private static File updateYarnConfFile(File confFile, File tempDir) { Configuration conf = new Configuration(false); try { conf.addResource(confFile.toURI().toURL()); } catch (MalformedURLException e) { LOG.error("File {} is malformed.", confFile, e); throw Throwables.propagate(e); } String yarnAppClassPath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); // add the pwd/* at the beginning of classpath. so user's jar will take precedence and without this change, // job.jar will be at the beginning of the classpath, since job.jar has old guava version classes, // we want to add pwd/* before yarnAppClassPath = "$PWD/*," + yarnAppClassPath; conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, yarnAppClassPath); File newYarnConfFile = new File(tempDir, "yarn-site.xml"); try (FileOutputStream os = new FileOutputStream(newYarnConfFile)) { conf.writeXml(os); } catch (IOException e) { LOG.error("Problem creating and writing to temporary yarn-conf.xml conf file at {}", newYarnConfFile, e); throw Throwables.propagate(e); } return newYarnConfFile; }
From source file:co.cask.cdap.explore.service.ExploreServiceUtilsTest.java
License:Apache License
@Test public void hijackConfFileTest() throws Exception { Configuration conf = new Configuration(false); conf.set("foo", "bar"); Assert.assertEquals(1, conf.size()); File tempDir = tmpFolder.newFolder(); File confFile = tmpFolder.newFile("hive-site.xml"); try (FileOutputStream os = new FileOutputStream(confFile)) { conf.writeXml(os);// w w w .j a v a 2 s. co m } File newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir); conf = new Configuration(false); conf.addResource(newConfFile.toURI().toURL()); Assert.assertEquals(3, conf.size()); Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_USER_CLASSPATH_FIRST)); Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_CLASSLOADER)); Assert.assertEquals("bar", conf.get("foo")); // check yarn-site changes confFile = tmpFolder.newFile("yarn-site.xml"); conf = new YarnConfiguration(); try (FileOutputStream os = new FileOutputStream(confFile)) { conf.writeXml(os); } String yarnApplicationClassPath = "$PWD/*," + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir); conf = new Configuration(false); conf.addResource(newConfFile.toURI().toURL()); Assert.assertEquals(yarnApplicationClassPath, conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH)); // check mapred-site changes confFile = tmpFolder.newFile("mapred-site.xml"); conf = new YarnConfiguration(); try (FileOutputStream os = new FileOutputStream(confFile)) { conf.writeXml(os); } String mapredApplicationClassPath = "$PWD/*," + conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH); newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir); conf = new Configuration(false); conf.addResource(newConfFile.toURI().toURL()); Assert.assertEquals(mapredApplicationClassPath, conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH)); // Ensure conf files that are not hive-site.xml/mapred-site.xml/yarn-site.xml are unchanged confFile = tmpFolder.newFile("core-site.xml"); Assert.assertEquals(confFile, ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir)); }
From source file:co.cask.cdap.internal.app.runtime.batch.distributed.MapReduceContainerHelper.java
License:Apache License
/** * Returns a list of path to be used for the MapReduce framework classpath. * * @param hConf the configuration for the job. * @param result a list for appending MR framework classpath * @return the same {@code result} list from the argument *//*from w w w .j a v a 2 s . c o m*/ public static List<String> getMapReduceClassPath(Configuration hConf, List<String> result) { String framework = hConf.get(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); // For classpath config get from the hConf, we splits it with both "," and ":" because one can set // the conf with something like "path1,path2:path3" and // it should become "path1:path2:path3" in the target JVM process Splitter splitter = Splitter.on(Pattern.compile(",|" + File.pathSeparatorChar)).trimResults() .omitEmptyStrings(); // If MR framework is non specified, use yarn.application.classpath and mapreduce.application.classpath // Otherwise, only use the mapreduce.application.classpath if (framework == null) { String yarnClassPath = hConf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); Iterables.addAll(result, splitter.split(yarnClassPath)); } // Add MR application classpath Iterables.addAll(result, splitter.split(hConf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))); return result; }
From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceRuntimeService.java
License:Apache License
@Override protected void startUp() throws Exception { // Creates a temporary directory locally for storing all generated files. File tempDir = createTempDirectory(); cleanupTask = createCleanupTask(tempDir); try {/*from ww w . j a v a 2s .com*/ Job job = createJob(new File(tempDir, "mapreduce")); Configuration mapredConf = job.getConfiguration(); classLoader = new MapReduceClassLoader(injector, cConf, mapredConf, context.getProgram().getClassLoader(), context.getPlugins(), context.getPluginInstantiator()); cleanupTask = createCleanupTask(cleanupTask, classLoader); mapredConf.setClassLoader(new WeakReferenceDelegatorClassLoader(classLoader)); ClassLoaders.setContextClassLoader(mapredConf.getClassLoader()); context.setJob(job); beforeSubmit(job); // Localize additional resources that users have requested via BasicMapReduceContext.localize methods Map<String, String> localizedUserResources = localizeUserResources(job, tempDir); // Override user-defined job name, since we set it and depend on the name. // https://issues.cask.co/browse/CDAP-2441 String jobName = job.getJobName(); if (!jobName.isEmpty()) { LOG.warn("Job name {} is being overridden.", jobName); } job.setJobName(getJobName(context)); // Create a temporary location for storing all generated files through the LocationFactory. Location tempLocation = createTempLocationDirectory(); cleanupTask = createCleanupTask(cleanupTask, tempLocation); // For local mode, everything is in the configuration classloader already, hence no need to create new jar if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // After calling beforeSubmit, we know what plugins are needed for the program, hence construct the proper // ClassLoader from here and use it for setting up the job Location pluginArchive = createPluginArchive(tempLocation); if (pluginArchive != null) { job.addCacheArchive(pluginArchive.toURI()); mapredConf.set(Constants.Plugin.ARCHIVE, pluginArchive.getName()); } } // set resources for the job TaskType.MAP.setResources(mapredConf, context.getMapperResources()); TaskType.REDUCE.setResources(mapredConf, context.getReducerResources()); // replace user's Mapper & Reducer's with our wrappers in job config MapperWrapper.wrap(job); ReducerWrapper.wrap(job); // packaging job jar which includes cdap classes with dependencies File jobJar = buildJobJar(job, tempDir); job.setJar(jobJar.toURI().toString()); Location programJar = programJarLocation; if (!MapReduceTaskContextProvider.isLocal(mapredConf)) { // Copy and localize the program jar in distributed mode programJar = copyProgramJar(tempLocation); job.addCacheFile(programJar.toURI()); List<String> classpath = new ArrayList<>(); // Localize logback.xml Location logbackLocation = createLogbackJar(tempLocation); if (logbackLocation != null) { job.addCacheFile(logbackLocation.toURI()); classpath.add(logbackLocation.getName()); } // Generate and localize the launcher jar to control the classloader of MapReduce containers processes classpath.add("job.jar/lib/*"); classpath.add("job.jar/classes"); Location launcherJar = createLauncherJar( Joiner.on(",").join(MapReduceContainerHelper.getMapReduceClassPath(mapredConf, classpath)), tempLocation); job.addCacheFile(launcherJar.toURI()); // The only thing in the container classpath is the launcher.jar // The MapReduceContainerLauncher inside the launcher.jar will creates a MapReduceClassLoader and launch // the actual MapReduce AM/Task from that // We explicitly localize the mr-framwork, but not use it with the classpath URI frameworkURI = MapReduceContainerHelper.getFrameworkURI(mapredConf); if (frameworkURI != null) { job.addCacheArchive(frameworkURI); } mapredConf.unset(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH); mapredConf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, launcherJar.getName()); mapredConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, launcherJar.getName()); } MapReduceContextConfig contextConfig = new MapReduceContextConfig(mapredConf); // We start long-running tx to be used by mapreduce job tasks. Transaction tx = txClient.startLong(); try { // We remember tx, so that we can re-use it in mapreduce tasks CConfiguration cConfCopy = cConf; contextConfig.set(context, cConfCopy, tx, programJar.toURI(), localizedUserResources); LOG.info("Submitting MapReduce Job: {}", context); // submits job and returns immediately. Shouldn't need to set context ClassLoader. job.submit(); this.job = job; this.transaction = tx; } catch (Throwable t) { Transactions.invalidateQuietly(txClient, tx); throw t; } } catch (Throwable t) { LOG.error("Exception when submitting MapReduce Job: {}", context, t); cleanupTask.run(); throw t; } }
From source file:co.cask.cdap.internal.app.runtime.distributed.AbstractDistributedProgramRunner.java
License:Apache License
@Override public final ProgramController run(final Program program, final ProgramOptions oldOptions) { final String schedulerQueueName = oldOptions.getArguments() .getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE); final File tempDir = DirUtils.createTempDir( new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)) .getAbsoluteFile()); try {/* w w w .ja va2 s. c o m*/ if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) { hConf.set(JobContext.QUEUE_NAME, schedulerQueueName); LOG.info("Setting scheduler queue to {}", schedulerQueueName); } Map<String, LocalizeResource> localizeResources = new HashMap<>(); final ProgramOptions options = addArtifactPluginFiles(oldOptions, localizeResources, DirUtils.createTempDir(tempDir)); // Copy config files and program jar to local temp, and ask Twill to localize it to container. // What Twill does is to save those files in HDFS and keep using them during the lifetime of application. // Twill will manage the cleanup of those files in HDFS. localizeResources.put("hConf.xml", new LocalizeResource(saveHConf(hConf, File.createTempFile("hConf", ".xml", tempDir)))); localizeResources.put("cConf.xml", new LocalizeResource(saveCConf(cConf, File.createTempFile("cConf", ".xml", tempDir)))); final URI logbackURI = getLogBackURI(program, tempDir); final String programOptions = GSON.toJson(options); // Obtains and add the HBase delegation token as well (if in non-secure mode, it's a no-op) // Twill would also ignore it if it is not running in secure mode. // The HDFS token should already obtained by Twill. return launch(program, options, localizeResources, tempDir, new ApplicationLauncher() { @Override public TwillController launch(TwillApplication twillApplication, Iterable<String> extraClassPaths, Iterable<? extends Class<?>> extraDependencies) { TwillPreparer twillPreparer = twillRunner.prepare(twillApplication); // TODO: CDAP-5506. It's a bit hacky to set a Spark environment here. However, we always launch // Spark using YARN and it is needed for both Workflow and Spark runner. We need to set it // because inside Spark code, it will set and unset the SPARK_YARN_MODE system properties, causing // fork in distributed mode not working. Setting it in the environment, which Spark uses for defaults, // so it can't be unset by Spark twillPreparer.withEnv(Collections.singletonMap("SPARK_YARN_MODE", "true")); if (options.isDebug()) { LOG.info("Starting {} with debugging enabled, programOptions: {}, and logback: {}", program.getId(), programOptions, logbackURI); twillPreparer.enableDebugging(); } // Add scheduler queue name if defined if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) { LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName); twillPreparer.setSchedulerQueue(schedulerQueueName); } if (logbackURI != null) { twillPreparer.withResources(logbackURI); } String logLevelConf = cConf.get(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL).toUpperCase(); if ("OFF".equals(logLevelConf)) { twillPreparer.addJVMOptions("-Dtwill.disable.kafka=true"); } else { LogEntry.Level logLevel = LogEntry.Level.ERROR; if ("ALL".equals(logLevelConf)) { logLevel = LogEntry.Level.TRACE; } else { try { logLevel = LogEntry.Level.valueOf(logLevelConf.toUpperCase()); } catch (Exception e) { LOG.warn("Invalid application container log level {}. Defaulting to ERROR.", logLevelConf); } } twillPreparer.addLogHandler(new ApplicationLogHandler( new PrinterLogHandler(new PrintWriter(System.out)), logLevel)); } String yarnAppClassPath = hConf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); // Add secure tokens if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) { // TokenSecureStoreUpdater.update() ignores parameters twillPreparer.addSecureStore(secureStoreUpdater.update(null, null)); } Iterable<Class<?>> dependencies = Iterables.concat( Collections.singletonList(HBaseTableUtilFactory.getHBaseTableUtilClass()), extraDependencies); twillPreparer.withDependencies(dependencies) .withClassPaths(Iterables.concat(extraClassPaths, Splitter.on(',').trimResults() .split(hConf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, "")))) .withApplicationClassPaths(Splitter.on(",").trimResults().split(yarnAppClassPath)) .withBundlerClassAcceptor(new HadoopClassExcluder() { @Override public boolean accept(String className, URL classUrl, URL classPathUrl) { // Exclude both hadoop and spark classes. return super.accept(className, classUrl, classPathUrl) && !className.startsWith("org.apache.spark."); } }).withApplicationArguments(String.format("--%s", RunnableOptions.JAR), program.getJarLocation().getName(), String.format("--%s", RunnableOptions.PROGRAM_OPTIONS), programOptions); TwillController twillController; // Change the context classloader to the combine classloader of this ProgramRunner and // all the classloaders of the dependencies classes so that Twill can trace classes. ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader( AbstractDistributedProgramRunner.this.getClass().getClassLoader(), Iterables.transform(dependencies, new Function<Class<?>, ClassLoader>() { @Override public ClassLoader apply(Class<?> input) { return input.getClassLoader(); } }))); try { twillController = twillPreparer.start(); } finally { ClassLoaders.setContextClassLoader(oldClassLoader); } return addCleanupListener(twillController, program, tempDir); } }); } catch (Exception e) { deleteDirectory(tempDir); throw Throwables.propagate(e); } }
From source file:com.bigjob.Client.java
License:Apache License
/** * Main run function for the client//from w w w.j a v a 2 s . c om * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM (RM)" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path // if (dfsUrl!=null && dfsUrl.equals("")==false){ // conf.set("fs.defaultFS", dfsUrl); // } FileSystem fs = FileSystem.get(conf); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(), localResources, null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.getId() + "/" + (Shell.WINDOWS ? windowBatPath : linuxShellPath); Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } if (!shellCommand.isEmpty()) { addToLocalResources(fs, null, shellCommandPath, appId.getId(), localResources, shellCommand); } if (shellArgs.length > 0) { addToLocalResources(fs, null, shellArgsPath, appId.getId(), localResources, StringUtils.join(shellArgs, " ")); } // Set local resource info into app master container launch context amContainer.setLocalResources(localResources); // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); amContainer.setEnvironment(env); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_containers " + String.valueOf(numContainers)); vargs.add("--priority " + String.valueOf(shellCmdPriority)); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); amContainer.setCommands(commands); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); capability.setVirtualCores(amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application //return monitorApplication(appId); System.out.println("ApplicationId:" + appId); return true; }