List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APPLICATION_CLASSPATH
String YARN_APPLICATION_CLASSPATH
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APPLICATION_CLASSPATH.
Click Source Link
From source file:org.apache.reef.runtime.yarn.YarnClasspathProvider.java
License:Apache License
@Inject YarnClasspathProvider(final YarnConfiguration yarnConfiguration) { boolean needsLegacyClasspath = false; // will be set to true below whenever we encounter issues with the YARN Configuration final ClassPathBuilder builder = new ClassPathBuilder(); try {/*w ww. j a va 2s .c o m*/ // Add the classpath actually configured on this cluster final String[] yarnClassPath = yarnConfiguration .getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH); if (null == yarnClassPath || yarnClassPath.length == 0) { needsLegacyClasspath = true; } else { builder.addAll(yarnClassPath); } final String[] yarnDefaultClassPath = YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH; if (null == yarnDefaultClassPath || yarnDefaultClassPath.length == 0) { LOG.log(Level.SEVERE, "YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH is empty. This indicates a broken cluster configuration"); needsLegacyClasspath = true; } else { builder.addAll(yarnDefaultClassPath); } } catch (final NoSuchFieldError e) { // This means that one of the static fields above aren't actually in YarnConfiguration. // The reason for that is most likely that we encounter a really old version of YARN. needsLegacyClasspath = true; LOG.log(Level.SEVERE, YARN_TOO_OLD_MESSAGE); } if (needsLegacyClasspath) { builder.addAll(LEGACY_CLASSPATH_LIST); } this.classPathPrefix = builder.getPrefixAsImmutableList(); this.classPathSuffix = builder.getSuffixAsImmutableList(); this.logClasspath(); }
From source file:org.apache.rya.periodic.notification.twill.yarn.PeriodicNotificationTwillRunner.java
License:Apache License
/** * Start an instance of the {@link PeriodicNotificationTwillApp}. * * @param interactive - If true, this method will block until the user terminates this JVM, at which point the * {@link PeriodicNotificationTwillApp} on the YARN cluster will also be terminated. If false, this * method will return after startup. *//*from www .j a v a 2 s . co m*/ public void startApp(final boolean interactive) { final String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); final List<String> applicationClassPaths = Lists.newArrayList(); Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath)); final TwillController controller = twillRunner.prepare(new PeriodicNotificationTwillApp(configFile)) .addLogHandler(new PrinterLogHandler( new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true))) .withApplicationClassPaths(applicationClassPaths) //.withApplicationArguments(args) //.withArguments(runnableName, args) // .withBundlerClassAcceptor(new HadoopClassExcluder()) .start(); final ResourceReport r = getResourceReport(controller, 5, TimeUnit.MINUTES); LOG.info("Received ResourceReport: {}", r); LOG.info("{} started successfully!", PeriodicNotificationTwillApp.APPLICATION_NAME); if (interactive) { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { LOG.info("{} waiting termination by user. Type ctrl-c to terminate.", PeriodicNotificationTwillApp.class.getSimpleName()); controller.awaitTerminated(); } catch (final ExecutionException e) { e.printStackTrace(); } } }
From source file:org.apache.slider.core.launch.ClasspathConstructor.java
License:Apache License
/** * Get the list of JARs from the YARN settings * @param config configuration/* w w w. j a va 2 s . com*/ */ public List<String> yarnApplicationClasspath(Configuration config) { String[] cp = config.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH); return cp != null ? Arrays.asList(cp) : new ArrayList<String>(0); }
From source file:org.apache.sysml.yarn.DMLYarnClient.java
License:Apache License
private static Map<String, String> constructEnvionmentMap(YarnConfiguration yconf) throws IOException { Map<String, String> eMap = new HashMap<>(); //setup default app master environment StringBuilder classpath = new StringBuilder(); for (String value : yconf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { if (classpath.length() > 0) classpath.append(File.pathSeparator); classpath.append(value.trim());/*from w w w. ja v a2 s.co m*/ } //setup mapreduce appmaster environment (for robustness if not included in default environment) //for example, by default HDP 2.2 did not include mapred client libraries in this configuration //note: we cannot use MRConfigurationNames.MR_APPLICATION_CLASSPATH because it refers to HDFS and $PWD that needs to be setup Map<String, String> env = System.getenv(); String mapred_home = null; //get mapred home via alternative environment variables if (env.containsKey(MAPRED_HOME_ENV_CONST)) { mapred_home = env.get(MAPRED_HOME_ENV_CONST); } else if (env.containsKey(HADOOP_HOME_ENV_CONST)) { String tmp = env.get(HADOOP_HOME_ENV_CONST); mapred_home = tmp + File.separator + ".." + File.separator + "hadoop-mapreduce"; } //concatenate mapred home libs to classpath if (mapred_home != null) { if (classpath.length() > 0) classpath.append(File.pathSeparator); classpath.append(mapred_home + File.separator + "*"); classpath.append(File.pathSeparator); classpath.append(mapred_home + File.separator + "lib" + File.separator + "*"); } eMap.put(Environment.CLASSPATH.name(), classpath.toString()); MRApps.setClasspath(eMap, yconf); LOG.debug("Constructed environment classpath: " + classpath.toString()); return eMap; }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
License:Apache License
private void setupEnv(ContainerLaunchContext amContainer) throws IOException { LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar) .append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim());//from w w w. ja v a2s .c om } classPathEnv.append(File.pathSeparatorChar).append("./").append(libDir).append("/*"); classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); env.put(Constants.TAJO_ARCHIVE_PATH, tajoArchive); env.put(Constants.TAJO_ARCHIVE_ROOT, "tajo/" + getTajoRootInArchive(tajoArchive)); env.put(Constants.TAJO_HOME, "$PWD/${" + Constants.TAJO_ARCHIVE_ROOT + "}"); env.put(Constants.TAJO_CONF_DIR, "$PWD/conf"); env.put(Constants.TAJO_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); env.put(Constants.TAJO_CLASSPATH, "/export/apps/hadoop/site/lib/*:$PWD/" + libDir + "/*"); amContainer.setEnvironment(env); }
From source file:org.apache.tez.common.TezYARNUtils.java
License:Apache License
public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) { StringBuilder classpathBuilder = new StringBuilder(); // Add any additional user-specified classpath String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX); if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) { classpathBuilder.append(additionalClasspath).append(File.pathSeparator); }//from www. jav a 2 s .c o m // Add PWD:PWD/* classpathBuilder.append(Environment.PWD.$()).append(File.pathSeparator) .append(Environment.PWD.$() + File.separator + "*").append(File.pathSeparator); // Next add the tez libs, if specified via an archive. if (usingArchive) { // Add PWD/tezlib/* classpathBuilder.append(Environment.PWD.$()).append(File.separator).append(TezConstants.TEZ_TAR_LR_NAME) .append(File.separator).append("*").append(File.pathSeparator); // Add PWD/tezlib/lib/* classpathBuilder.append(Environment.PWD.$()).append(File.separator).append(TezConstants.TEZ_TAR_LR_NAME) .append(File.separator).append("lib").append(File.separator).append("*") .append(File.pathSeparator); } // Last add HADOOP_CLASSPATH, if it's required. if (conf.getBoolean(TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS, TezConfiguration.TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT)) { for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classpathBuilder.append(c.trim()).append(File.pathSeparator); } } else { // Setup HADOOP_CONF_DIR after PWD and tez-libs, if it's required. classpathBuilder.append(Environment.HADOOP_CONF_DIR.$()).append(File.pathSeparator); } String classpath = classpathBuilder.toString(); return StringInterner.weakIntern(classpath); }
From source file:org.apache.tez.test.MiniTezCluster.java
License:Apache License
@Override public void serviceStart() throws Exception { LOG.info("Starting MiniTezCluster"); super.serviceStart(); File workDir = super.getTestWorkDir(); Configuration conf = super.getConfig(); confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG); File confFile = new File(confFilePath.toString()); try {//from w ww . j av a2 s. c o m confFile.createNewFile(); conf.writeXml(new FileOutputStream(confFile)); confFile.deleteOnExit(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); throw new RuntimeException(e); } confFilePath = new Path(confFile.getAbsolutePath()); conf.setStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, workDir.getAbsolutePath(), System.getProperty("java.class.path")); LOG.info("Setting yarn-site.xml via YARN-APP-CP at: " + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH)); }
From source file:org.apache.tez.tests.MiniTezClusterWithTimeline.java
License:Apache License
@Override public void serviceStart() throws Exception { LOG.info("Starting MiniTezClusterWithTimeline"); super.serviceStart(); File workDir = super.getTestWorkDir(); Configuration conf = super.getConfig(); confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG); File confFile = new File(confFilePath.toString()); try {/* w ww. ja va2 s.co m*/ confFile.createNewFile(); conf.writeXml(new FileOutputStream(confFile)); confFile.deleteOnExit(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); throw new RuntimeException(e); } confFilePath = new Path(confFile.getAbsolutePath()); conf.setStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, workDir.getAbsolutePath(), System.getProperty("java.class.path")); LOG.info("Setting yarn-site.xml via YARN-APP-CP at: " + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH)); }
From source file:org.apache.twill.example.yarn.HelloWorld.java
License:Apache License
public static void main(String[] args) { if (args.length < 1) { System.err.println("Arguments format: <host:port of zookeeper server>"); System.exit(1);/* w ww . j ava 2 s . c o m*/ } String zkStr = args[0]; YarnConfiguration yarnConfiguration = new YarnConfiguration(); final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr); twillRunner.start(); String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH, Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)); List<String> applicationClassPaths = Lists.newArrayList(); Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath)); final TwillController controller = twillRunner.prepare(new HelloWorldRunnable()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationClassPaths(applicationClassPaths) .withBundlerClassAcceptor(new HadoopClassExcluder()).start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { Futures.getUnchecked(controller.terminate()); } finally { twillRunner.stop(); } } }); try { controller.awaitTerminated(); } catch (ExecutionException e) { e.printStackTrace(); } }
From source file:org.conan.myhadoop02.mr.yarntest.Client.java
License:Apache License
/** * Main run function for the client/*w w w. java 2s . c om*/ * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } if (domainId != null && domainId.length() > 0 && toCreateDomain) { prepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // The shell script has to be made available on the final container(s) // where it will be executed. // To do this, we need to first copy into the filesystem that is visible // to the yarn framework. // We do not need to set this as a local resource for the application // master as the application master does not need it. String hdfsShellScriptLocation = ""; long hdfsShellScriptLen = 0; long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); String shellPathSuffix = appName + "/" + appId.toString() + "/" + SCRIPT_PATH; Path shellDst = new Path(fs.getHomeDirectory(), shellPathSuffix); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); hdfsShellScriptLen = shellFileStatus.getLen(); hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); } if (!shellCommand.isEmpty()) { addToLocalResources(fs, null, shellCommandPath, appId.toString(), localResources, shellCommand); } if (shellArgs.length > 0) { addToLocalResources(fs, null, shellArgsPath, appId.toString(), localResources, StringUtils.join(shellArgs, " ")); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); if (domainId != null && domainId.length() > 0) { env.put(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN, domainId); } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add("--container_memory " + String.valueOf(containerMemory)); vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); vargs.add("--num_containers " + String.valueOf(numContainers)); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } vargs.add("--priority " + String.valueOf(shellCmdPriority)); for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } if (debugFlag) { vargs.add("--debug"); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // amContainer.setServiceData(serviceData); // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // TODO // Try submitting the same request again // app submission failure? // Monitor the application return monitorApplication(appId); }