Example usage for org.apache.hadoop.yarn.conf HAUtil isHAEnabled

List of usage examples for org.apache.hadoop.yarn.conf HAUtil isHAEnabled

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.conf.*
  5. HAUtil
  6. isHAEnabled

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf HAUtil isHAEnabled.

Prototype

public static boolean isHAEnabled(Configuration conf)

Source Link

Document

Returns true if Resource Manager HA is configured.

Usage

From source file:co.cask.cdap.common.security.YarnTokenUtils.java

License:Apache License

/**
 * Gets a Yarn delegation token and stores it in the given Credentials.
 *
 * @return the same Credentials instance as the one given in parameter.
 *///www.java2s.  co m
public static Credentials obtainToken(YarnConfiguration configuration, Credentials credentials) {
    if (!UserGroupInformation.isSecurityEnabled()) {
        return credentials;
    }

    try {
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(configuration);
        yarnClient.start();

        try {
            Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName());
            org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient
                    .getRMDelegationToken(renewer);

            // TODO: The following logic should be replaced with call to ClientRMProxy.getRMDelegationTokenService after
            // CDAP-4825 is resolved
            List<String> services = new ArrayList<>();
            if (HAUtil.isHAEnabled(configuration)) {
                // If HA is enabled, we need to enumerate all RM hosts
                // and add the corresponding service name to the token service
                // Copy the yarn conf since we need to modify it to get the RM addresses
                YarnConfiguration yarnConf = new YarnConfiguration(configuration);
                for (String rmId : HAUtil.getRMHAIds(configuration)) {
                    yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
                    InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
                            YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
                    services.add(SecurityUtil.buildTokenService(address).toString());
                }
            } else {
                services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString());
            }

            Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken,
                    (InetSocketAddress) null);
            token.setService(new Text(Joiner.on(',').join(services)));
            credentials.addToken(new Text(token.getService()), token);

            // OK to log, it won't log the credential, only information about the token.
            LOG.info("Added RM delegation token: {}", token);

        } finally {
            yarnClient.stop();
        }

        return credentials;
    } catch (Exception e) {
        LOG.error("Failed to get secure token for Yarn.", e);
        throw Throwables.propagate(e);
    }
}

From source file:io.hops.util.GroupMembershipService.java

License:Apache License

@Override
public synchronized void serviceInit(Configuration conf) throws Exception {

    this.conf = conf;
    groupMembershipServiceAddress = conf.getSocketAddr(YarnConfiguration.RM_BIND_HOST,
            YarnConfiguration.RM_GROUP_MEMBERSHIP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_GROUP_MEMBERSHIP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_GROUP_MEMBERSHIP_PORT);
    adminAcl = new AccessControlList(
            conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
    if (HAUtil.isHAEnabled(conf)) {
        this.rmId = HAUtil.getRMHAId(conf);
    }// w ww.  j  a va 2  s  .  c o  m
    daemonUser = UserGroupInformation.getCurrentUser();
    authorizer = YarnAuthorizationProvider.getInstance(conf);
    authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation.getCurrentUser());

    LOG.info("init groupMembershipService " + this.rmId);
}

From source file:org.apache.kylin.tool.common.HadoopConfExtractor.java

License:Apache License

public static String extractYarnMasterUrl(Configuration conf) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http(s)?://)([^:]*):([^/])*.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        if (m.matches()) {
            return m.group(1) + m.group(2) + ":" + m.group(3);
        }//www.j a  va  2  s .c  o m
    }

    logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");

    String webappConfKey, defaultAddr;
    if (YarnConfiguration.useHttps(conf)) {
        webappConfKey = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS;
        defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS;
    } else {
        webappConfKey = YarnConfiguration.RM_WEBAPP_ADDRESS;
        defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS;
    }

    String rmWebHost;
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(webappConfKey, active), defaultAddr,
                yarnConf);
    } else {
        rmWebHost = HAUtil.getConfValueForRMInstance(webappConfKey, defaultAddr, conf);
    }

    if (StringUtils.isEmpty(rmWebHost)) {
        return null;
    }
    if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
        rmWebHost = (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
    logger.info("yarn master url: " + rmWebHost);
    return rmWebHost;
}

From source file:org.apache.kylin.tool.JobTaskCounterExtractor.java

License:Apache License

private String getRestCheckUrl() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http://)(.*):.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        m.matches();//from ww  w .j a  va 2  s  .c  o m
        yarnUrl = m.group(1) + m.group(2) + ":19888";
        return yarnUrl;
    } else {
        logger.info("kylin.job.yarn.app.rest.check.status.url" + " is not set read from hadoop configuration");
    }
    Configuration conf = HadoopUtil.getCurrentConfiguration();
    String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(
                HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active),
                YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
    }
    if (StringUtils.isEmpty(rmWebHost)) {
        return null;
    }
    if (rmWebHost.startsWith("http://") || rmWebHost.startsWith("https://")) {
        //do nothing
    } else {
        rmWebHost = "http://" + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    m.matches();
    return m.group(1) + m.group(2) + ":19888";
}

From source file:org.apache.kylin.tool.MrJobInfoExtractor.java

License:Apache License

private void extractRestCheckUrl() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    final String yarnStatusCheckUrl = config.getYarnStatusCheckUrl();
    Pattern pattern = Pattern.compile("(http://)([^:]*):([^/])*.*");
    if (yarnStatusCheckUrl != null) {
        Matcher m = pattern.matcher(yarnStatusCheckUrl);
        if (m.matches()) {
            jobHistoryUrlBase = m.group(1) + m.group(2) + ":19888";
            yarnMasterUrlBase = m.group(1) + m.group(2) + ":" + m.group(3);
        }/*from  w w w  .j ava2 s.  c  om*/
    }
    logger.info("kylin.engine.mr.yarn-check-status-url" + " is not set, read from hadoop configuration");

    Configuration conf = HadoopUtil.getCurrentConfiguration();
    String rmWebHost = HAUtil.getConfValueForRMInstance(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, conf);
    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        String active = RMHAUtils.findActiveRMHAId(yarnConf);
        rmWebHost = HAUtil.getConfValueForRMInstance(
                HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, active),
                YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, yarnConf);
    }
    if (StringUtils.isEmpty(rmWebHost)) {
        return;
    }
    if (!rmWebHost.startsWith("http://") && !rmWebHost.startsWith("https://")) {
        rmWebHost = "http://" + rmWebHost;
    }
    Matcher m = pattern.matcher(rmWebHost);
    Preconditions.checkArgument(m.matches(), "Yarn master URL not found.");
    yarnMasterUrlBase = rmWebHost;
    jobHistoryUrlBase = m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address",
            m.group(2) + ":19888", conf);
}

From source file:org.apache.slider.core.launch.CredentialUtils.java

License:Apache License

/**
 * Look up and return the resource manager's principal. This method
 * automatically does the <code>_HOST</code> replacement in the principal and
 * correctly handles HA resource manager configurations.
 *
 * From: YARN-4629/* ww w. j  a  va2  s . c  o m*/
 * @param conf the {@link Configuration} file from which to read the
 * principal
 * @return the resource manager's principal string
 * @throws IOException thrown if there's an error replacing the host name
 */
public static String getRMPrincipal(Configuration conf) throws IOException {
    String principal = conf.get(RM_PRINCIPAL, "");
    String hostname;
    Preconditions.checkState(!principal.isEmpty(), "Not set: " + RM_PRINCIPAL);

    if (HAUtil.isHAEnabled(conf)) {
        YarnConfiguration yarnConf = new YarnConfiguration(conf);
        if (yarnConf.get(RM_HA_ID) == null) {
            // If RM_HA_ID is not configured, use the first of RM_HA_IDS.
            // Any valid RM HA ID should work.
            String[] rmIds = yarnConf.getStrings(RM_HA_IDS);
            Preconditions.checkState((rmIds != null) && (rmIds.length > 0), "Not set " + RM_HA_IDS);
            yarnConf.set(RM_HA_ID, rmIds[0]);
        }

        hostname = yarnConf.getSocketAddr(RM_ADDRESS, DEFAULT_RM_ADDRESS, DEFAULT_RM_PORT).getHostName();
    } else {
        hostname = conf.getSocketAddr(RM_ADDRESS, DEFAULT_RM_ADDRESS, DEFAULT_RM_PORT).getHostName();
    }
    return SecurityUtil.getServerPrincipal(principal, hostname);
}

From source file:org.apache.twill.internal.yarn.Hadoop23YarnAppClient.java

License:Apache License

/**
 * Overrides parent method to adds RM delegation token to the given context. If YARN is running with HA RM,
 * delegation tokens for each RM service will be added.
 *///from www  . jav  a2s  .c  o m
protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) {
    if (!UserGroupInformation.isSecurityEnabled()) {
        return;
    }

    try {
        Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName());
        org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient.getRMDelegationToken(renewer);

        // The following logic is copied from ClientRMProxy.getRMDelegationTokenService, which is not available in
        // YARN older than 2.4
        List<String> services = new ArrayList<>();
        if (HAUtil.isHAEnabled(configuration)) {
            // If HA is enabled, we need to enumerate all RM hosts
            // and add the corresponding service name to the token service
            // Copy the yarn conf since we need to modify it to get the RM addresses
            YarnConfiguration yarnConf = new YarnConfiguration(configuration);
            for (String rmId : HAUtil.getRMHAIds(configuration)) {
                yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
                InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
                        YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
                services.add(SecurityUtil.buildTokenService(address).toString());
            }
        } else {
            services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString());
        }

        Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());

        // casting needed for later Hadoop version
        @SuppressWarnings("RedundantCast")
        Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken,
                (InetSocketAddress) null);

        token.setService(new Text(Joiner.on(',').join(services)));
        credentials.addToken(new Text(token.getService()), token);

        LOG.debug("Added RM delegation token {} for application {}", token, appId);
        credentials.addToken(token.getService(), token);

        context.setTokens(YarnUtils.encodeCredentials(credentials));

    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}