Example usage for org.apache.hadoop.yarn.conf HAUtil getConfValueForRMInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf HAUtil getConfValueForRMInstance.

Prototype

public static String getConfValueForRMInstance(String prefix, String defaultValue, Configuration conf) 

Source Link

Usage

From source file:org.apache.kylin.tool.common.HadoopConfExtractor.java

License:Apache License

public static String extractYarnMasterUrl(Configuration conf) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http(s)?://)([^:]*):([^/])*.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        if (m.matches()) {
            return m.group(1) + m.group(2) + ":" + m.group(3);
        }//from   w  ww  .ja  v  a2 s.c  o m
    }

    logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");

    String webappConfKey, defaultAddr;
    if (YarnConfiguration.useHttps(conf)) {
        webappConfKey = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS;
        defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS;
    } else {
        webappConfKey = YarnConfiguration.RM_WEBAPP_ADDRESS;
        defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS;
    }

    String rmWebHost;
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(webappConfKey, active), defaultAddr,
                yarnConf);
    } else {
        rmWebHost = HAUtil.getConfValueForRMInstance(webappConfKey, defaultAddr, conf);
    }

    if (StringUtils.isEmpty(rmWebHost)) {
        return null;
    }
    if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
        rmWebHost = (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
    logger.info("yarn master url: " + rmWebHost);
    return rmWebHost;
}

---

From source file:org.apache.kylin.tool.common.HadoopConfExtractor.java

License:Apache License

public static String extractJobHistoryUrl(String yarnWebapp, Configuration conf) {
    Pattern pattern = Pattern.compile("(http(s)?://)([^:]*):([^/])*.*");
    Matcher m = pattern.matcher(yarnWebapp);
    Preconditions.checkArgument(m.matches(), "Yarn master URL" + yarnWebapp + " not right.");
    return m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address",
            m.group(2) + ":19888", conf);
}

---

From source file:org.apache.kylin.tool.JobTaskCounterExtractor.java

License:Apache License

private String getRestCheckUrl() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http://)(.*):.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        m.matches();//from w w w . j a v a2 s  .com
        yarnUrl = m.group(1) + m.group(2) + ":19888";
        return yarnUrl;
    } else {
        logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not set read from hadoop configuration");
    }
    Configuration conf = HadoopUtil.getCurrentConfiguration();
    String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(
                HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active),
                YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
    }
    if (StringUtils.isEmpty(rmWebHost)) {
        return null;
    }
    if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) {
        //do nothing
    } else {
        rmWebHost = "http://" + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    m.matches();
    return m.group(1) + m.group(2) + ":19888";
}

---

From source file:org.apache.kylin.tool.MrJobInfoExtractor.java

License:Apache License

private void extractRestCheckUrl() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http://)([^:]*):([^/])*.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        if (m.matches()) {
            jobHistoryUrlBase = m.group(1) + m.group(2) + ":19888";
            yarnMasterUrlBase = m.group(1) + m.group(2) + ":" + m.group(3);
        }/* w ww.j  ava 2s.  com*/
    }
    logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");

    Configuration conf = HadoopUtil.getCurrentConfiguration();
    String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(
                HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active),
                YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
    }
    if (StringUtils.isEmpty(rmWebHost)) {
        return;
    }
    if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
        rmWebHost = "http://" + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
    yarnMasterUrlBase = rmWebHost;
    jobHistoryUrlBase = m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address",
            m.group(2) + ":19888", conf);
}

---