List of usage examples for org.apache.hadoop.yarn.conf HAUtil addSuffix
public static String addSuffix(String key, String suffix)
From source file:co.cask.cdap.operations.yarn.YarnRMHAOperationalStatsTest.java
License:Apache License
@Override protected MiniYARNCluster createYarnCluster() throws IOException, InterruptedException, YarnException { Configuration hConf = new Configuration(); hConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); String hostname = MiniYARNCluster.getHostname(); for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { hConf.set(HAUtil.addSuffix(confKey, "rm0"), hostname + ":" + Networks.getRandomPort()); hConf.set(HAUtil.addSuffix(confKey, "rm1"), hostname + ":" + Networks.getRandomPort()); }/*from w ww. java 2s . c o m*/ MiniYARNCluster yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 2, 2, 2); yarnCluster.init(hConf); yarnCluster.start(); yarnCluster.getResourceManager(0).getRMContext().getRMAdminService().transitionToActive( new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER)); return yarnCluster; }
From source file:org.apache.kylin.tool.common.HadoopConfExtractor.java
License:Apache License
public static String extractYarnMasterUrl(Configuration conf) { KylinConfig config = KylinConfig.getInstanceFromEnv(); final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl(); Pattern pattern = Pattern.compile("(http(s)?://)([^:]*):([^/])*.*"); if (yarnStatusCheckUrl != null) { Matcher m = pattern.matcher(yarnStatusCheckUrl); if (m.matches()) { return m.group(1) + m.group(2) + ":" + m.group(3); }/*w ww . j a v a 2 s .c om*/ } logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration"); String webappConfKey, defaultAddr; if (YarnConfiguration.useHttps(conf)) { webappConfKey = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS; defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS; } else { webappConfKey = YarnConfiguration.RM_WEBAPP_ADDRESS; defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS; } String rmWebHost; if (HAUtil.isHAEnabled(conf)) { YarnConfiguration yarnConf = new YarnConfiguration(conf); String active = RMHAUtils.findActiveRMHAId(yarnConf); rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(webappConfKey, active), defaultAddr, yarnConf); } else { rmWebHost = HAUtil.getConfValueForRMInstance(webappConfKey, defaultAddr, conf); } if (StringUtils.isEmpty(rmWebHost)) { return null; } if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) { rmWebHost = (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + rmWebHost; } Matcher m = pattern.matcher(rmWebHost); Preconditions.checkArgument(m.matches(), "Yarn master URL not found."); logger.info("yarn master url: " + rmWebHost); return rmWebHost; }
From source file:org.apache.kylin.tool.JobTaskCounterExtractor.java
License:Apache License
private String getRestCheckUrl() { KylinConfig config = KylinConfig.getInstanceFromEnv(); final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl(); Pattern pattern = Pattern.compile("(http://)(.*):.*"); if (yarnStatusCheckUrl != null) { Matcher m = pattern.matcher(yarnStatusCheckUrl); m.matches();//from w w w . j a v a 2 s . c om yarnUrl = m.group(1) + m.group(2) + ":19888"; return yarnUrl; } else { logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not set read from hadoop configuration"); } Configuration conf = HadoopUtil.getCurrentConfiguration(); String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf); if (HAUtil.isHAEnabled(conf)) { YarnConfiguration yarnConf = new YarnConfiguration(conf); String active = RMHAUtils.findActiveRMHAId(yarnConf); rmWebHost = HAUtil.getConfValueForRMInstance( HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf); } if (StringUtils.isEmpty(rmWebHost)) { return null; } if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) { //do nothing } else { rmWebHost = "http://" + rmWebHost; } Matcher m = pattern.matcher(rmWebHost); m.matches(); return m.group(1) + m.group(2) + ":19888"; }
From source file:org.apache.kylin.tool.MrJobInfoExtractor.java
License:Apache License
private void extractRestCheckUrl() { KylinConfig config = KylinConfig.getInstanceFromEnv(); final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl(); Pattern pattern = Pattern.compile("(http://)([^:]*):([^/])*.*"); if (yarnStatusCheckUrl != null) { Matcher m = pattern.matcher(yarnStatusCheckUrl); if (m.matches()) { jobHistoryUrlBase = m.group(1) + m.group(2) + ":19888"; yarnMasterUrlBase = m.group(1) + m.group(2) + ":" + m.group(3); }//from w w w .j ava 2s. co m } logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration"); Configuration conf = HadoopUtil.getCurrentConfiguration(); String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf); if (HAUtil.isHAEnabled(conf)) { YarnConfiguration yarnConf = new YarnConfiguration(conf); String active = RMHAUtils.findActiveRMHAId(yarnConf); rmWebHost = HAUtil.getConfValueForRMInstance( HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf); } if (StringUtils.isEmpty(rmWebHost)) { return; } if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) { rmWebHost = "http://" + rmWebHost; } Matcher m = pattern.matcher(rmWebHost); Preconditions.checkArgument(m.matches(), "Yarn master URL not found."); yarnMasterUrlBase = rmWebHost; jobHistoryUrlBase = m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address", m.group(2) + ":19888", conf); }