Example usage for org.apache.hadoop.security UserGroupInformation addToken

List of usage examples for org.apache.hadoop.security UserGroupInformation addToken

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. UserGroupInformation
  6. addToken

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation addToken.

Prototype

public boolean addToken(Token<? extends TokenIdentifier> token)

Source Link

Document

Add a token to this UGI

Usage

From source file:org.apache.falcon.catalog.HiveCatalogService.java

License:Apache License

private static void addSecureCredentialsAndToken(Configuration conf, HiveConf hcatConf,
        UserGroupInformation proxyUGI) throws IOException {
    if (UserGroupInformation.isSecurityEnabled()) {
        String metaStoreServicePrincipal = conf.get(SecurityUtil.HIVE_METASTORE_KERBEROS_PRINCIPAL);
        hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, metaStoreServicePrincipal);
        hcatConf.set(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, "true");

        Token<DelegationTokenIdentifier> delegationTokenId = getDelegationToken(hcatConf,
                metaStoreServicePrincipal);
        proxyUGI.addToken(delegationTokenId);
    }/*from w  w  w  . j ava2  s.c  o m*/
}

From source file:org.apache.flink.mesos.runtime.clusterframework.MesosTaskManagerRunner.java

License:Apache License

public static void runTaskManager(String[] args, final Class<? extends TaskManager> taskManager)
        throws IOException {
    EnvironmentInformation.logEnvironmentInfo(LOG, taskManager.getSimpleName(), args);
    org.apache.flink.runtime.util.SignalHandler.register(LOG);

    // try to parse the command line arguments
    final Configuration configuration;
    try {/*from  ww w.  j  a  v  a  2 s  .co  m*/
        configuration = TaskManager.parseArgsAndLoadConfig(args);

        // add dynamic properties to TaskManager configuration.
        final Configuration dynamicProperties = FlinkMesosSessionCli
                .decodeDynamicProperties(ENV.get(MesosConfigKeys.ENV_DYNAMIC_PROPERTIES));
        LOG.debug("Mesos dynamic properties: {}", dynamicProperties);
        configuration.addAll(dynamicProperties);
    } catch (Throwable t) {
        LOG.error("Failed to load the TaskManager configuration and dynamic properties.", t);
        System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
        return;
    }

    // read the environment variables
    final Map<String, String> envs = System.getenv();
    final String effectiveUsername = envs.get(MesosConfigKeys.ENV_CLIENT_USERNAME);
    final String tmpDirs = envs.get(MesosConfigKeys.ENV_FLINK_TMP_DIR);

    // configure local directory
    String flinkTempDirs = configuration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null);
    if (flinkTempDirs != null) {
        LOG.info(
                "Overriding Mesos temporary file directories with those " + "specified in the Flink config: {}",
                flinkTempDirs);
    } else if (tmpDirs != null) {
        LOG.info("Setting directories for temporary files to: {}", tmpDirs);
        configuration.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tmpDirs);
    }

    LOG.info("Mesos task runs as '{}', setting user to execute Flink TaskManager to '{}'",
            UserGroupInformation.getCurrentUser().getShortUserName(), effectiveUsername);

    // tell akka to die in case of an error
    configuration.setBoolean(ConfigConstants.AKKA_JVM_EXIT_ON_FATAL_ERROR, true);

    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(effectiveUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);
    }

    // Infer the resource identifier from the environment variable
    String containerID = Preconditions.checkNotNull(envs.get(MesosConfigKeys.ENV_FLINK_CONTAINER_ID));
    final ResourceID resourceId = new ResourceID(containerID);
    LOG.info("ResourceID assigned for this container: {}", resourceId);

    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                TaskManager.selectNetworkInterfaceAndRunTaskManager(configuration, resourceId, taskManager);
            } catch (Throwable t) {
                LOG.error("Error while starting the TaskManager", t);
                System.exit(TaskManager.STARTUP_FAILURE_RETURN_CODE());
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    final String yarnClientUsername = System.getenv(Client.ENV_CLIENT_USERNAME);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink ApplicationMaster/JobManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);
    }//from  w ww.  j  a v a 2  s. c om
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                new ApplicationMaster().run();
            } catch (Exception e) {
                e.printStackTrace();
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.appMaster.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    // execute Application Master using the client's user
    final String yarnClientUsername = System.getenv(Client.ENV_CLIENT_USERNAME);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink ApplicationMaster/JobManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);
    }//from  w ww .ja v a 2  s.c  o  m
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            AMRMClient<ContainerRequest> rmClient = null;
            ApplicationMaster am = null;
            try {
                Configuration conf = Utils.initializeYarnConfiguration();
                rmClient = AMRMClient.createAMRMClient();
                rmClient.init(conf);
                rmClient.start();

                // run the actual Application Master
                am = new ApplicationMaster(conf);
                am.generateConfigurationFile();
                am.startJobManager();
                am.setRMClient(rmClient);
                am.run();
            } catch (Throwable e) {
                LOG.error("Error while running the application master", e);
                // the AM is not available. Report error through the unregister function.
                if (rmClient != null && am == null) {
                    try {
                        rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED,
                                "Flink YARN Application master" + " stopped unexpectedly with an exception.\n"
                                        + StringUtils.stringifyException(e),
                                "");
                    } catch (Exception e1) {
                        LOG.error("Unable to fail the application master", e1);
                    }
                    LOG.info("AM unregistered from RM");
                    return null;
                }
                if (rmClient == null) {
                    LOG.error("Unable to unregister AM since the RM client is not available");
                }
                if (am != null) {
                    LOG.info("Writing error into internal message system");
                    am.setFailed(true);
                    am.addMessage(new Message("The application master failed with an exception:\n"
                            + StringUtils.stringifyException(e)));
                    am.keepRPCAlive();
                }
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.appMaster.YarnTaskManagerRunner.java

License:Apache License

public static void main(final String[] args) throws IOException {
    Map<String, String> envs = System.getenv();
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final String localDirs = envs.get(Environment.LOCAL_DIRS.key());

    // configure local directory
    final String[] newArgs = Arrays.copyOf(args, args.length + 2);
    newArgs[newArgs.length - 2] = "-" + TaskManager.ARG_CONF_DIR;
    newArgs[newArgs.length - 1] = localDirs;
    LOG.info("Setting log path " + localDirs);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink TaskManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);
    }/*from   w ww . j  a  v a  2 s .c  o m*/
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                TaskManager.main(newArgs);
            } catch (Exception e) {
                LOG.error("Error while running the TaskManager", e);
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.YarnApplicationMasterRunner.java

License:Apache License

/**
 * The instance entry point for the YARN application master. Obtains user group
 * information and calls the main work method {@link #runApplicationMaster()} as a
 * privileged action./*w  ww. j a  va  2s  .  co m*/
 *
 * @param args The command line arguments.
 * @return The process exit code.
 */
protected int run(String[] args) {
    try {
        LOG.debug("All environment variables: {}", ENV);

        final String yarnClientUsername = ENV.get(YarnConfigKeys.ENV_CLIENT_USERNAME);
        require(yarnClientUsername != null, "YARN client user name environment variable {} not set",
                YarnConfigKeys.ENV_CLIENT_USERNAME);

        final UserGroupInformation currentUser;
        try {
            currentUser = UserGroupInformation.getCurrentUser();
        } catch (Throwable t) {
            throw new Exception("Cannot access UserGroupInformation information for current user", t);
        }

        LOG.info("YARN daemon runs as user {}. Running Flink Application Master/JobManager as user {}",
                currentUser.getShortUserName(), yarnClientUsername);

        UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);

        // transfer all security tokens, for example for authenticated HDFS and HBase access
        for (Token<?> token : currentUser.getTokens()) {
            ugi.addToken(token);
        }

        // run the actual work in a secured privileged action
        return ugi.doAs(new PrivilegedAction<Integer>() {
            @Override
            public Integer run() {
                return runApplicationMaster();
            }
        });
    } catch (Throwable t) {
        // make sure that everything whatever ends up in the log
        LOG.error("YARN Application Master initialization failed", t);
        return INIT_ERROR_EXIT_CODE;
    }
}

From source file:org.apache.flink.yarn.YarnTaskManagerRunner.java

License:Apache License

public static void main(final String[] args) throws IOException {
    Map<String, String> envs = System.getenv();
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final String localDirs = envs.get(Environment.LOCAL_DIRS.key());

    // configure local directory
    final String[] newArgs = Arrays.copyOf(args, args.length + 2);
    newArgs[newArgs.length - 2] = "-" + TaskManager.ARG_CONF_DIR;
    newArgs[newArgs.length - 1] = localDirs;
    LOG.info("Setting log path " + localDirs);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink TaskManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);
    }/*from w ww  .  j  av a  2 s . c o m*/
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                TaskManager.main(newArgs);
            } catch (Exception e) {
                e.printStackTrace();
            }
            return null;
        }
    });
}

From source file:org.apache.gobblin.util.hadoop.TokenUtils.java

License:Apache License

/**
 *
 * @param userToProxy The user that hiveClient is impersonating as to fetch the delegation tokens.
 * @param ugi The {@link UserGroupInformation} that to be added with negotiated credentials.
 */// w w  w . jav a  2s  . com
public static void getHiveToken(final State state, IMetaStoreClient hiveClient, Credentials cred,
        final String userToProxy, UserGroupInformation ugi) {
    try {
        // Fetch the delegation token with "service" field overwritten with the metastore.uri configuration.
        // org.apache.gobblin.hive.HiveMetaStoreClientFactory.getHiveConf(com.google.common.base.Optional<java.lang.String>)
        // sets the signature field to the same value to retrieve the token correctly.
        HiveConf hiveConf = new HiveConf();
        Token<DelegationTokenIdentifier> hcatToken = fetchHcatToken(userToProxy, hiveConf,
                hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname), hiveClient);
        cred.addToken(hcatToken.getService(), hcatToken);
        ugi.addToken(hcatToken);

        // Fetch extra Hcat location user specified.
        final List<String> extraHcatLocations = state.contains(USER_DEFINED_HIVE_LOCATIONS)
                ? state.getPropAsList(USER_DEFINED_HIVE_LOCATIONS)
                : Collections.EMPTY_LIST;
        if (!extraHcatLocations.isEmpty()) {
            LOG.info("Need to fetch extra metaStore tokens from hive.");

            // start to process the user inputs.
            for (final String thriftUrl : extraHcatLocations) {
                LOG.info("Fetching metaStore token from : " + thriftUrl);

                hiveConf = new HiveConf();
                hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, thriftUrl);
                hcatToken = fetchHcatToken(userToProxy, hiveConf, thriftUrl, hiveClient);
                cred.addToken(hcatToken.getService(), hcatToken);
                ugi.addToken(hcatToken);

                LOG.info("Successfully fetched token for:" + thriftUrl);
            }
        }
    } catch (final Throwable t) {
        final String message = "Failed to get hive metastore token." + t.getMessage() + t.getCause();
        LOG.error(message, t);
        throw new RuntimeException(message);
    }
}

From source file:org.apache.hama.bsp.BSPApplicationMaster.java

License:Apache License

/**
 * Connects to the Resource Manager./*from w  w  w  .  j  a  va  2 s  .c o m*/
 * 
 * @param yarnConf
 * @return a new RPC connection to the Resource Manager.
 */
private ApplicationMasterProtocol getYarnRPCConnection(Configuration yarnConf) throws IOException {
    // Connect to the Scheduler of the ResourceManager.
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    final InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf
            .get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));

    Token<? extends TokenIdentifier> amRMToken = setupAndReturnAMRMToken(rmAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);

    final Configuration conf = yarnConf;

    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) yarnRPC.getProxy(ApplicationMasterProtocol.class, rmAddress,
                    conf);
        }
    });
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    return client;
}

From source file:org.apache.hama.bsp.JobImpl.java

License:Apache License

/**
 *
 * @param rpc/*w w  w .j av  a2s. co  m*/
 * @param nmToken
 * @param nodeId
 * @param user
 * @return
 */
protected ContainerManagementProtocol getContainerManagementProtocolProxy(final YarnRPC rpc, Token nmToken,
        NodeId nodeId, String user) {
    ContainerManagementProtocol proxy;
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    final InetSocketAddress addr = NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort());
    if (nmToken != null) {
        ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
    }

    proxy = ugi.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
        @Override
        public ContainerManagementProtocol run() {
            return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, addr, conf);
        }
    });
    return proxy;
}