Example usage for org.apache.hadoop.security UserGroupInformation createRemoteUser

List of usage examples for org.apache.hadoop.security UserGroupInformation createRemoteUser

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.security.*
  5. UserGroupInformation
  6. createRemoteUser

Introduction

In this page you can find the example usage for org.apache.hadoop.security UserGroupInformation createRemoteUser.

Prototype

@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation createRemoteUser(String user)

Source Link

Document

Create a user from a login name.

Usage

From source file:org.apache.flink.runtime.clusterframework.overlays.HadoopUserOverlayTest.java

License:Apache License

@Test
public void testConfigure() throws Exception {

    final UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test");

    HadoopUserOverlay overlay = new HadoopUserOverlay(ugi);

    ContainerSpecification spec = new ContainerSpecification();
    overlay.configure(spec);//from   w  w  w .  ja  v a 2  s  .c o m

    assertEquals(ugi.getUserName(), spec.getEnvironmentVariables().get("HADOOP_USER_NAME"));
}

From source file:org.apache.flink.runtime.clusterframework.overlays.HadoopUserOverlayTest.java

License:Apache License

@Test
public void testBuilderFromEnvironment() throws Exception {

    final Configuration conf = new Configuration();
    final UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test");

    ugi.doAs(new PrivilegedAction<Object>() {
        @Override//from   w  ww  .  j  a v  a 2s . co m
        public Object run() {
            try {
                HadoopUserOverlay.Builder builder = HadoopUserOverlay.newBuilder().fromEnvironment(conf);
                assertEquals(ugi, builder.ugi);
                return null;
            } catch (Exception ex) {
                throw new AssertionError(ex);
            }
        }
    });
}

From source file:org.apache.flink.yarn.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    final String yarnClientUsername = System.getenv(Client.ENV_CLIENT_USERNAME);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink ApplicationMaster/JobManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);//from   w ww .  j  a  va2s . c o m
    }
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                new ApplicationMaster().run();
            } catch (Exception e) {
                e.printStackTrace();
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.appMaster.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws Exception {
    // execute Application Master using the client's user
    final String yarnClientUsername = System.getenv(Client.ENV_CLIENT_USERNAME);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink ApplicationMaster/JobManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);//from  w  w  w .  j  a v  a  2s  . c  o m
    }
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            AMRMClient<ContainerRequest> rmClient = null;
            ApplicationMaster am = null;
            try {
                Configuration conf = Utils.initializeYarnConfiguration();
                rmClient = AMRMClient.createAMRMClient();
                rmClient.init(conf);
                rmClient.start();

                // run the actual Application Master
                am = new ApplicationMaster(conf);
                am.generateConfigurationFile();
                am.startJobManager();
                am.setRMClient(rmClient);
                am.run();
            } catch (Throwable e) {
                LOG.error("Error while running the application master", e);
                // the AM is not available. Report error through the unregister function.
                if (rmClient != null && am == null) {
                    try {
                        rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED,
                                "Flink YARN Application master" + " stopped unexpectedly with an exception.\n"
                                        + StringUtils.stringifyException(e),
                                "");
                    } catch (Exception e1) {
                        LOG.error("Unable to fail the application master", e1);
                    }
                    LOG.info("AM unregistered from RM");
                    return null;
                }
                if (rmClient == null) {
                    LOG.error("Unable to unregister AM since the RM client is not available");
                }
                if (am != null) {
                    LOG.info("Writing error into internal message system");
                    am.setFailed(true);
                    am.addMessage(new Message("The application master failed with an exception:\n"
                            + StringUtils.stringifyException(e)));
                    am.keepRPCAlive();
                }
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.appMaster.YarnTaskManagerRunner.java

License:Apache License

public static void main(final String[] args) throws IOException {
    Map<String, String> envs = System.getenv();
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final String localDirs = envs.get(Environment.LOCAL_DIRS.key());

    // configure local directory
    final String[] newArgs = Arrays.copyOf(args, args.length + 2);
    newArgs[newArgs.length - 2] = "-" + TaskManager.ARG_CONF_DIR;
    newArgs[newArgs.length - 1] = localDirs;
    LOG.info("Setting log path " + localDirs);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink TaskManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);//from ww  w  . j av a 2  s .  co m
    }
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                TaskManager.main(newArgs);
            } catch (Exception e) {
                LOG.error("Error while running the TaskManager", e);
            }
            return null;
        }
    });
}

From source file:org.apache.flink.yarn.YarnApplicationMasterRunner.java

License:Apache License

/**
 * The instance entry point for the YARN application master. Obtains user group
 * information and calls the main work method {@link #runApplicationMaster()} as a
 * privileged action./*from   www . j  ava 2 s .com*/
 *
 * @param args The command line arguments.
 * @return The process exit code.
 */
protected int run(String[] args) {
    try {
        LOG.debug("All environment variables: {}", ENV);

        final String yarnClientUsername = ENV.get(YarnConfigKeys.ENV_CLIENT_USERNAME);
        require(yarnClientUsername != null, "YARN client user name environment variable {} not set",
                YarnConfigKeys.ENV_CLIENT_USERNAME);

        final UserGroupInformation currentUser;
        try {
            currentUser = UserGroupInformation.getCurrentUser();
        } catch (Throwable t) {
            throw new Exception("Cannot access UserGroupInformation information for current user", t);
        }

        LOG.info("YARN daemon runs as user {}. Running Flink Application Master/JobManager as user {}",
                currentUser.getShortUserName(), yarnClientUsername);

        UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);

        // transfer all security tokens, for example for authenticated HDFS and HBase access
        for (Token<?> token : currentUser.getTokens()) {
            ugi.addToken(token);
        }

        // run the actual work in a secured privileged action
        return ugi.doAs(new PrivilegedAction<Integer>() {
            @Override
            public Integer run() {
                return runApplicationMaster();
            }
        });
    } catch (Throwable t) {
        // make sure that everything whatever ends up in the log
        LOG.error("YARN Application Master initialization failed", t);
        return INIT_ERROR_EXIT_CODE;
    }
}

From source file:org.apache.flink.yarn.YarnTaskManagerRunner.java

License:Apache License

public static void main(final String[] args) throws IOException {
    Map<String, String> envs = System.getenv();
    final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME);
    final String localDirs = envs.get(Environment.LOCAL_DIRS.key());

    // configure local directory
    final String[] newArgs = Arrays.copyOf(args, args.length + 2);
    newArgs[newArgs.length - 2] = "-" + TaskManager.ARG_CONF_DIR;
    newArgs[newArgs.length - 1] = localDirs;
    LOG.info("Setting log path " + localDirs);
    LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting"
            + " user to execute Flink TaskManager to '" + yarnClientUsername + "'");
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername);
    for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) {
        ugi.addToken(toks);/*ww w .  j a v  a  2 s.  c  o  m*/
    }
    ugi.doAs(new PrivilegedAction<Object>() {
        @Override
        public Object run() {
            try {
                TaskManager.main(newArgs);
            } catch (Exception e) {
                e.printStackTrace();
            }
            return null;
        }
    });
}

From source file:org.apache.gobblin.yarn.YarnServiceTest.java

License:Apache License

private void startApp() throws Exception {
    // submit a dummy app
    ApplicationSubmissionContext appSubmissionContext = yarnClient.createApplication()
            .getApplicationSubmissionContext();
    this.applicationId = appSubmissionContext.getApplicationId();

    ContainerLaunchContext containerLaunchContext = BuilderUtils.newContainerLaunchContext(
            Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "100"),
            Collections.emptyMap(), null, Collections.emptyMap());

    // Setup the application submission context
    appSubmissionContext.setApplicationName("TestApp");
    appSubmissionContext.setResource(Resource.newInstance(128, 1));
    appSubmissionContext.setPriority(Priority.newInstance(0));
    appSubmissionContext.setAMContainerSpec(containerLaunchContext);

    this.yarnClient.submitApplication(appSubmissionContext);

    // wait for application to be accepted
    int i;/*w ww  .j a va2 s  .c om*/
    RMAppAttempt attempt = null;
    for (i = 0; i < 120; i++) {
        ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

        if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
            this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
            attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
                    .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
            break;
        }
        Thread.sleep(1000);
    }

    Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

    // Set the AM-RM token in the UGI for access during testing
    UserGroupInformation.setLoginUser(
            UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
    UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}

From source file:org.apache.hama.bsp.ApplicationMaster.java

License:Apache License

/**
 * Main run function for the application master
 * //from w w w.  j a  va2s  . co m
 * @throws org.apache.hadoop.yarn.exceptions.YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException, InterruptedException {
    LOG.info("Starting ApplicationMaster");

    // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
    // are marked as LimitedPrivate
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    // Now remove the AM->RM token so that containers cannot access it.
    Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
    LOG.info("Executing with tokens:");
    while (iter.hasNext()) {
        Token<?> token = iter.next();
        LOG.info(token);
        if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
            iter.remove();
        }
    }
    allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    // Create appSubmitterUgi and add original tokens to it
    String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name());
    appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName);
    appSubmitterUgi.addCredentials(credentials);

    AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
    amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
    amRMClient.init(localConf);
    amRMClient.start();

    containerListener = createNMCallbackHandler();
    nmClientAsync = new NMClientAsyncImpl(containerListener);
    nmClientAsync.init(localConf);
    nmClientAsync.start();

    // Setup local RPC Server to accept status requests directly from clients
    // TODO need to setup a protocol for client to be able to communicate to
    // the RPC server
    // TODO use the rpc port info to register with the RM for the client to
    // send requests to this app master

    // Register self with ResourceManager
    // This will start heartbeating to the RM
    appMasterHostname = NetUtils.getHostname();
    RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname,
            appMasterRpcPort, appMasterTrackingUrl);
    // Dump out information about cluster capability as seen by the
    // resource manager
    int maxMem = response.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capability of resources in this cluster " + maxMem);

    int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
    LOG.info("Max vcores capability of resources in this cluster " + maxVCores);

    // A resource ask cannot exceed the max.
    if (containerMemory > maxMem) {
        LOG.info("Container memory specified above max threshold of cluster." + " Using max value."
                + ", specified=" + containerMemory + ", max=" + maxMem);
        containerMemory = maxMem;
    }

    if (containerVirtualCores > maxVCores) {
        LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value."
                + ", specified=" + containerVirtualCores + ", max=" + maxVCores);
        containerVirtualCores = maxVCores;
    }

    List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts();
    LOG.info(appAttemptID + " received " + previousAMRunningContainers.size()
            + " previous attempts' running containers on AM registration.");
    for (Container container : previousAMRunningContainers) {
        launchedContainers.add(container.getId());
    }
    numAllocatedContainers.addAndGet(previousAMRunningContainers.size());

    int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size();
    // Setup ask for containers from RM
    // Send request for containers to RM
    // Until we get our fully allocated quota, we keep on polling RM for
    // containers
    // Keep looping until all the containers are launched and shell script
    // executed on them ( regardless of success/failure).
    for (int i = 0; i < numTotalContainersToRequest; ++i) {
        AMRMClient.ContainerRequest containerAsk = setupContainerAskForRM();
        amRMClient.addContainerRequest(containerAsk);
    }
    numRequestedContainers.set(numTotalContainers);
}

From source file:org.apache.hama.bsp.BSPApplicationMaster.java

License:Apache License

/**
 * Connects to the Resource Manager./*  ww  w  . j  a v a 2  s .c  o  m*/
 * 
 * @param yarnConf
 * @return a new RPC connection to the Resource Manager.
 */
private ApplicationMasterProtocol getYarnRPCConnection(Configuration yarnConf) throws IOException {
    // Connect to the Scheduler of the ResourceManager.
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    final InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf
            .get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));

    Token<? extends TokenIdentifier> amRMToken = setupAndReturnAMRMToken(rmAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);

    final Configuration conf = yarnConf;

    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) yarnRPC.getProxy(ApplicationMasterProtocol.class, rmAddress,
                    conf);
        }
    });
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    return client;
}