List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_ADDRESS
String RM_SCHEDULER_ADDRESS
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_ADDRESS.
Click Source Link
From source file:org.apache.hoya.yarn.client.HoyaClient.java
License:Apache License
/** * * @param clustername// w ww . jav a2s .c o m * @param clusterDirectory * @param instanceDefinition * @param debugAM * @return the launched application * @throws YarnException * @throws IOException */ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { deployedClusterName = clustername; HoyaUtils.validateClusterName(clustername); verifyNoLiveClusters(clustername); Configuration config = getConfig(); boolean clusterSecure = HoyaUtils.isClusterSecure(config); //create the Hoya AM provider -this helps set up the AM HoyaAMClientProvider hoyaAM = new HoyaAMClientProvider(config); instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist( internalOptions.getMandatoryOption(OptionKeys.INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist( internalOptions.getMandatoryOption(OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( internalOptions.getMandatoryOption(OptionKeys.INTERNAL_PROVIDER_NAME)); // make sure the conf dir is valid; // now build up the image path // TODO: consider supporting apps that don't have an image path Path imagePath = HoyaUtils.extractImagePath(hoyaFileSystem, internalOptions); if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } MapOperations hoyaAMResourceComponent = resourceOperations.getOrAddComponent(HoyaKeys.COMPONENT_AM); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, HoyaKeys.APP_TYPE, config, hoyaFileSystem, yarnClient, clusterSecure, hoyaAMResourceComponent); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); amLauncher.setMaxAppAttempts(config.getInt(KEY_AM_RESTART_LIMIT, DEFAULT_AM_RESTART_LIMIT)); hoyaFileSystem.purgeHoyaAppInstanceTempFiles(clustername); Path tempPath = hoyaFileSystem.createHoyaAppInstanceTempPath(clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); hoyaFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", hoyaFileSystem.toString(), tempPath, libPath); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); // conf directory setup Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = System.getProperty(HoyaKeys.PROPERTY_CONF_DIR); if (confdirProp == null || confdirProp.isEmpty()) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); if (!confDir.exists()) { throw new BadConfigException(HOYA_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } Path localConfDirPath = HoyaUtils.createLocalPath(confDir); log.debug("Copying AM configuration data from {}", localConfDirPath); remoteConfPath = new Path(clusterDirectory, HoyaKeys.SUBMITTED_CONF_DIR); HoyaUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself if (!getUsingMiniMRCluster()) { log.debug("Destination is not a MiniYARNCluster -copying full classpath"); // insert conf dir first if (remoteConfPath != null) { relativeConfDir = HoyaKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = hoyaFileSystem.submitDirectory(remoteConfPath, relativeConfDir); HoyaUtils.mergeMaps(localResources, submittedConfDir); } log.debug("Copying JARs from local filesystem"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path HoyaUtils.putJar(localResources, hoyaFileSystem, this.getClass(), tempPath, libdir, SLIDER_JAR); } // build up the configuration // IMPORTANT: it is only after this call that site configurations // will be valid. propagatePrincipals(config, instanceDefinition); Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); HoyaUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); // add AM and provider specific artifacts to the resource map Map<String, LocalResource> providerResources; // standard AM resources hoyaAM.prepareAMAndConfigForLaunch(hoyaFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath); //add provider-specific resources provider.prepareAMAndConfigForLaunch(hoyaFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath); // now that the site config is fully generated, the provider gets // to do a quick review of them. log.debug("Preflight validation of cluster configuration"); hoyaAM.preflightValidateClusterConfiguration(hoyaFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); provider.preflightValidateClusterConfiguration(hoyaFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); // now add the image if it was set if (hoyaFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv(HoyaUtils.buildEnvMap(hoyaAMResourceComponent)); String classpath = HoyaUtils.buildClasspath(relativeConfDir, libdir, getConfig(), getUsingMiniMRCluster()); amLauncher.setEnv("CLASSPATH", classpath); if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", HoyaUtils.stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", hoyaFileSystem.listFSDir(libPath)); } // rm address InetSocketAddress rmSchedulerAddress = null; try { rmSchedulerAddress = HoyaUtils.getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS, config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); CommandLineBuilder commandLine = new CommandLineBuilder(); commandLine.addJavaBinary(); // insert any JVM options); hoyaAM.addJVMOptions(instanceDefinition, commandLine); // enable asserts if the text option is set commandLine.enableJavaAssertions(); // add the hoya AM sevice entry point commandLine.add(HoyaAMArgs.CLASSNAME); // create action and the cluster name commandLine.add(HoyaActions.ACTION_CREATE); commandLine.add(clustername); // debug if (debugAM) { commandLine.add(Arguments.ARG_DEBUG); } // set the cluster directory path commandLine.add(Arguments.ARG_HOYA_CLUSTER_URI); commandLine.add(clusterDirectory.toUri().toString()); if (!isUnset(rmAddr)) { commandLine.add(Arguments.ARG_RM_ADDR); commandLine.add(rmAddr); } if (serviceArgs.getFilesystemURL() != null) { commandLine.add(Arguments.ARG_FILESYSTEM); commandLine.add(serviceArgs.getFilesystemURL().toString()); } if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over propagateConfOption(commandLine, config, HoyaXmlConfKeys.KEY_SECURITY_ENABLED); propagateConfOption(commandLine, config, DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = commandLine.build(); log.info("Completed setting up app master command {}", cmdStr); amLauncher.addCommandLine(commandLine); // the Hoya AM gets to configure the AM requirements, not the custom provider hoyaAM.prepareAMResourceRequirements(hoyaAMResourceComponent, amLauncher.getResource()); // Set the priority for the application master int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY); amLauncher.setPriority(amPriority); // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_HOYA_YARN_QUEUE); amLauncher.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * * @param clustername name of the cluster * @param clusterDirectory cluster dir//w w w . j a v a2s. c o m * @param instanceDefinition the instance definition * @param debugAM enable debug AM options * @return the launched application * @throws YarnException * @throws IOException */ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { deployedClusterName = clustername; SliderUtils.validateClusterName(clustername); verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config); //create the Slider AM provider -this helps set up the AM SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME)); // make sure the conf dir is valid; if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, appOperations.getGlobalOptions().get(AgentKeys.APP_DEF)); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config, sliderFileSystem, yarnClient, clusterSecure, sliderAMResourceComponent, resourceGlobalOptions, applicationTags); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); amLauncher.setMaxAppAttempts(maxAppAttempts); sliderFileSystem.purgeAppInstanceTempFiles(clustername); Path tempPath = sliderFileSystem.createAppInstanceTempPath(clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); sliderFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(), tempPath, libPath); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); // look for the configuration directory named on the command line boolean hasServerLog4jProperties = false; Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); if (confdirProp == null || confdirProp.isEmpty()) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); if (!confDir.exists()) { throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } Path localConfDirPath = SliderUtils.createLocalPath(confDir); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath); SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); hasServerLog4jProperties = log4jserver.isFile(); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself boolean usingMiniMRCluster = getUsingMiniMRCluster(); if (!usingMiniMRCluster) { log.debug("Destination is not a MiniYARNCluster -copying full classpath"); // insert conf dir first if (remoteConfPath != null) { relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); SliderUtils.mergeMaps(localResources, submittedConfDir); } } // build up the configuration // IMPORTANT: it is only after this call that site configurations // will be valid. propagatePrincipals(config, instanceDefinition); // validate security data /* // turned off until tested SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, instanceDefinition, clustername); */ Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); // standard AM resources sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); //add provider-specific resources provider.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); // now that the site config is fully generated, the provider gets // to do a quick review of them. log.debug("Preflight validation of cluster configuration"); sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); provider.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); // TODO: consider supporting apps that don't have an image path Path imagePath = SliderUtils.extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv(SliderUtils.buildEnvMap(sliderAMResourceComponent)); ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(), usingMiniMRCluster); amLauncher.setClasspath(classpath); if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", SliderUtils.stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); } // rm address InetSocketAddress rmSchedulerAddress; try { rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS, config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder(); // insert any JVM options); sliderAM.addJVMOptions(instanceDefinition, commandLine); // enable asserts if the text option is set commandLine.enableJavaAssertions(); // if the conf dir has a log4j-server.properties, switch to that if (hasServerLog4jProperties) { commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); } // add the AM sevice entry point commandLine.add(SliderAppMaster.SERVICE_CLASSNAME); // create action and the cluster name commandLine.add(ACTION_CREATE, clustername); // debug if (debugAM) { commandLine.add(Arguments.ARG_DEBUG); } // set the cluster directory path commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri()); if (!isUnset(rmAddr)) { commandLine.add(Arguments.ARG_RM_ADDR, rmAddr); } if (serviceArgs.getFilesystemBinding() != null) { commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); } /** * pass the registry binding */ addConfOptionToCLI(commandLine, config, REGISTRY_PATH, DEFAULT_REGISTRY_PATH); addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over /* addConfOptionToCLI(commandLine, config, KEY_SECURITY); */ addConfOptionToCLI(commandLine, config, DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = commandLine.build(); log.debug("Completed setting up app master command {}", cmdStr); amLauncher.addCommandLine(commandLine); // the Slider AM gets to configure the AM requirements, not the custom provider sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, amLauncher.getResource()); // Set the priority for the application master int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY); amLauncher.setPriority(amPriority); // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE); if (!SliderUtils.isUnset(suppliedQueue)) { amQueue = suppliedQueue; log.info("Using queue {} for the application instance.", amQueue); } if (amQueue != null) { amLauncher.setQueue(amQueue); } // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; }
From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java
License:Apache License
@Override public void init(Configuration conf) { this.conf = conf; connectYarnClient();//from w ww . j a v a 2 s. c om final YarnConfiguration yarnConf = new YarnConfiguration(conf); final YarnRPC rpc = YarnRPC.create(conf); final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); UserGroupInformation currentUser; try { currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnRuntimeException(e); } rmClient = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() { @Override public ApplicationMasterProtocol run() { return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmAddress, yarnConf); } }); }
From source file:org.apache.tajo.MiniTajoYarnCluster.java
License:Apache License
@Override public void init(Configuration conf) { conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, new InetSocketAddress("127.0.0.1", 0)); conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, new InetSocketAddress("127.0.0.1", 0)); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath()); }//from w ww . j a v a 2 s . c o m conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); try { Path stagingPath = FileContext.getFileContext(conf) .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR))); FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); } LOG.info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.mkdir(stagingPath, null, true); //mkdir done directory as well String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.makeQualified(new Path(doneDir)); fc.mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, PullServerAuxService.PULLSERVER_SERVICEID); conf.setClass( String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, PullServerAuxService.PULLSERVER_SERVICEID), PullServerAuxService.class, Service.class); // Non-standard shuffle port conf.setInt(TajoConf.ConfVars.PULLSERVER_PORT.name(), 0); // local directory conf.set(TajoConf.ConfVars.WORKER_TEMPORAL_DIR.name(), "/tmp/tajo-localdir"); conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class, ContainerExecutor.class); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 600); super.init(conf); }
From source file:org.apache.twill.internal.appmaster.ApplicationMasterMain.java
License:Apache License
/** * Optionally sets the RM scheduler address based on the environment variable if it is not set in the cluster config. *///from w w w.j ava2 s. c o m private static void setRMSchedulerAddress(Configuration conf) { String schedulerAddress = System.getenv(EnvKeys.YARN_RM_SCHEDULER_ADDRESS); if (schedulerAddress == null) { return; } // If the RM scheduler address is not in the config or it's from yarn-default.xml, // replace it with the one from the env, which is the same as the one client connected to. String[] sources = conf.getPropertySources(YarnConfiguration.RM_SCHEDULER_ADDRESS); if (sources == null || sources.length == 0 || "yarn-default.xml".equals(sources[sources.length - 1])) { conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, schedulerAddress); } }
From source file:org.apache.twill.yarn.YarnTwillPreparer.java
License:Apache License
@Override public TwillController start() { try {/*from w ww . j av a 2 s . c o m*/ final ProcessLauncher<ApplicationMasterInfo> launcher = yarnAppClient.createLauncher(twillSpec, schedulerQueue); final ApplicationMasterInfo appMasterInfo = launcher.getContainerInfo(); Callable<ProcessController<YarnApplicationReport>> submitTask = new Callable<ProcessController<YarnApplicationReport>>() { @Override public ProcessController<YarnApplicationReport> call() throws Exception { String fsUser = locationFactory.getHomeLocation().getName(); // Local files needed by AM Map<String, LocalFile> localFiles = Maps.newHashMap(); // Local files declared by runnables Multimap<String, LocalFile> runnableLocalFiles = HashMultimap.create(); createAppMasterJar(createBundler(), localFiles); createContainerJar(createBundler(), localFiles); populateRunnableLocalFiles(twillSpec, runnableLocalFiles); saveSpecification(twillSpec, runnableLocalFiles, localFiles); saveLogback(localFiles); saveLauncher(localFiles); saveJvmOptions(localFiles); saveArguments(new Arguments(arguments, runnableArgs), localFiles); saveEnvironments(localFiles); saveLocalFiles(localFiles, ImmutableSet.of(Constants.Files.TWILL_SPEC, Constants.Files.LOGBACK_TEMPLATE, Constants.Files.CONTAINER_JAR, Constants.Files.LAUNCHER_JAR, Constants.Files.ARGUMENTS)); LOG.debug("Submit AM container spec: {}", appMasterInfo); // java -Djava.io.tmpdir=tmp -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory // org.apache.twill.internal.TwillLauncher // appMaster.jar // org.apache.twill.internal.appmaster.ApplicationMasterMain // false ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String>builder() .put(EnvKeys.TWILL_FS_USER, fsUser) .put(EnvKeys.TWILL_APP_DIR, getAppLocation().toURI().toASCIIString()) .put(EnvKeys.TWILL_ZK_CONNECT, zkConnectString).put(EnvKeys.TWILL_RUN_ID, runId.getId()) .put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory)) .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName()).put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); LOG.debug("Log level is set to {} for the Twill application.", logLevel); builder.put(EnvKeys.TWILL_APP_LOG_LEVEL, logLevel.toString()); int memory = Resources.computeMaxHeapSize(appMasterInfo.getMemoryMB(), Constants.APP_MASTER_RESERVED_MEMORY_MB, Constants.HEAP_MIN_RATIO); return launcher.prepareLaunch(builder.build(), localFiles.values(), credentials) .addCommand("$JAVA_HOME/bin/java", "-Djava.io.tmpdir=tmp", "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR, "-Dtwill.app=$" + EnvKeys.TWILL_APP_NAME, "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR", "-Xmx" + memory + "m", extraOptions == null ? "" : extraOptions, TwillLauncher.class.getName(), Constants.Files.APP_MASTER_JAR, ApplicationMasterMain.class.getName(), Boolean.FALSE.toString()) .launch(); } }; YarnTwillController controller = controllerFactory.create(runId, logHandlers, submitTask); controller.start(); return controller; } catch (Exception e) { LOG.error("Failed to submit application {}", twillSpec.getName(), e); throw Throwables.propagate(e); } }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java
License:Apache License
public AMRMProtocol getAMResourceManager() { if (amResourceManager != null) return amResourceManager; LOG.debug("Using configuration: " + conf); YarnConfiguration yarnConf = new YarnConfiguration(conf); YarnRPC rpc = YarnRPC.create(yarnConf); InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); LOG.info("Connecting to the resource manager (scheduling) at " + rmAddress); amResourceManager = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf); return amResourceManager; }
From source file:org.springframework.xd.sqoop.SqoopRunner.java
License:Apache License
protected static Configuration createConfiguration(Map<String, String> configOptions) { Configuration configuration = new Configuration(); setConfigurationProperty(configOptions, configuration, CommonConfigurationKeys.FS_DEFAULT_NAME_KEY); setConfigurationProperty(configOptions, configuration, YarnConfiguration.RM_HOSTNAME); setConfigurationProperty(configOptions, configuration, YarnConfiguration.RM_ADDRESS); setConfigurationProperty(configOptions, configuration, YarnConfiguration.RM_SCHEDULER_ADDRESS); setConfigurationProperty(configOptions, configuration, YarnConfiguration.YARN_APPLICATION_CLASSPATH); setConfigurationProperty(configOptions, configuration, "mapreduce.framework.name"); if (StringUtils.hasText(configOptions.get("mapreduce.jobhistory.address"))) { setConfigurationProperty(configOptions, configuration, "mapreduce.jobhistory.address"); }// w w w.ja v a2 s.c om if (configOptions.containsKey(SECURITY_AUTH_METHOD) && "kerberos".equals(configOptions.get(SECURITY_AUTH_METHOD))) { configuration.setBoolean("hadoop.security.authorization", true); configuration.set("hadoop.security.authentication", configOptions.get(SECURITY_AUTH_METHOD)); configuration.set("dfs.namenode.kerberos.principal", configOptions.get(SECURITY_NAMENODE_PRINCIPAL)); configuration.set("yarn.resourcemanager.principal", configOptions.get(SECURITY_RM_MANAGER_PRINCIPAL)); if (StringUtils.hasText(configOptions.get(SECURITY_MAPREDUCE_JOBHISTORY_PRINCIPAL))) { configuration.set("mapreduce.jobhistory.principal", configOptions.get(SECURITY_MAPREDUCE_JOBHISTORY_PRINCIPAL)); } String userKeytab = configOptions.get(SECURITY_USER_KEYTAB); String userPrincipal = configOptions.get(SECURITY_USER_PRINCIPAL); UserGroupInformation.setConfiguration(configuration); if (StringUtils.hasText(userKeytab)) { configuration.set(ConfigurationFactoryBean.USERKEYTAB, userKeytab.trim()); } if (StringUtils.hasText(userPrincipal)) { configuration.set(ConfigurationFactoryBean.USERPRINCIPAL, userPrincipal.trim()); } if (StringUtils.hasText(userKeytab) && StringUtils.hasText(userPrincipal)) { try { SecurityUtil.login(configuration, ConfigurationFactoryBean.USERKEYTAB, ConfigurationFactoryBean.USERPRINCIPAL); } catch (Exception e) { logger.warn("Cannot login using keytab " + userKeytab + " and principal " + userPrincipal, e); } } } for (Entry<String, String> entry : configOptions.entrySet()) { String key = entry.getKey(); if (key.startsWith(SPRING_HADOOP_CONFIG_PREFIX + ".")) { String prop = key.substring(SPRING_HADOOP_CONFIG_PREFIX.length() + 1); String value = entry.getValue(); logger.info("Setting configuration property: " + prop + "=" + value); configuration.set(prop, value); } } return configuration; }
From source file:org.springframework.yarn.am.AppmasterRmTemplate.java
License:Apache License
@Override protected InetSocketAddress getRpcAddress(Configuration config) { InetSocketAddress addr = config.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); try {/*from w ww .ja v a 2 s. com*/ setupTokens(addr); } catch (IOException e) { log.error("Error setting up tokens", e); } return addr; }
From source file:org.springframework.yarn.client.CommandYarnClient.java
License:Apache License
@Override public Map<String, String> getEnvironment() { // For the environment we set additional env variables // from configuration. This is most useful for unit // testing where resource manager and hdfs are created // dynamically within mini clusters. Map<String, String> env = super.getEnvironment(); env.put(YarnSystemConstants.RM_ADDRESS, getConfiguration().get(YarnConfiguration.RM_ADDRESS)); env.put(YarnSystemConstants.FS_ADDRESS, getConfiguration().get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)); env.put(YarnSystemConstants.SCHEDULER_ADDRESS, getConfiguration().get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); if (log.isDebugEnabled()) { log.debug("Setting additional env variables " + YarnSystemConstants.RM_ADDRESS + "=" + env.get(YarnSystemConstants.RM_ADDRESS) + YarnSystemConstants.FS_ADDRESS + "=" + env.get(YarnSystemConstants.FS_ADDRESS) + YarnSystemConstants.SCHEDULER_ADDRESS + "=" + env.get(YarnSystemConstants.SCHEDULER_ADDRESS)); }//from ww w. jav a2 s . c om return env; }