List of usage examples for org.apache.hadoop.security UserGroupInformation addToken
public boolean addToken(Token<? extends TokenIdentifier> token)
From source file:alluxio.yarn.ApplicationMaster.java
License:Apache License
/** * @param args Command line arguments to launch application master *//*from w w w . jav a 2 s . co m*/ public static void main(String[] args) { Options options = new Options(); options.addOption("num_workers", true, "Number of Alluxio workers to launch. Default 1"); options.addOption("master_address", true, "(Required) Address to run Alluxio master"); options.addOption("resource_path", true, "(Required) HDFS path containing the Application Master"); try { LOG.info("Starting Application Master with args {}", Arrays.toString(args)); final CommandLine cliParser = new GnuParser().parse(options, args); YarnConfiguration conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); if (UserGroupInformation.isSecurityEnabled()) { String user = System.getenv("ALLUXIO_USER"); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); for (Token token : UserGroupInformation.getCurrentUser().getTokens()) { ugi.addToken(token); } LOG.info("UserGroupInformation: " + ugi); ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { runApplicationMaster(cliParser); return null; } }); } else { runApplicationMaster(cliParser); } } catch (Exception e) { LOG.error("Error running Application Master", e); System.exit(1); } }
From source file:azkaban.jobtype.HadoopJavaJobRunnerMain.java
License:Apache License
public HadoopJavaJobRunnerMain() throws Exception { Runtime.getRuntime().addShutdownHook(new Thread() { @Override/*from w w w. jav a 2 s. co m*/ public void run() { cancelJob(); } }); try { _jobName = System.getenv(ProcessJob.JOB_NAME_ENV); String propsFile = System.getenv(ProcessJob.JOB_PROP_ENV); _logger = Logger.getRootLogger(); _logger.removeAllAppenders(); ConsoleAppender appender = new ConsoleAppender(DEFAULT_LAYOUT); appender.activateOptions(); _logger.addAppender(appender); _logger.setLevel(Level.INFO); //Explicitly setting level to INFO Properties props = new Properties(); props.load(new BufferedReader(new FileReader(propsFile))); HadoopConfigurationInjector.injectResources(new Props(null, props)); final Configuration conf = new Configuration(); UserGroupInformation.setConfiguration(conf); securityEnabled = UserGroupInformation.isSecurityEnabled(); _logger.info("Running job " + _jobName); String className = props.getProperty(JOB_CLASS); if (className == null) { throw new Exception("Class name is not set."); } _logger.info("Class name " + className); UserGroupInformation loginUser = null; UserGroupInformation proxyUser = null; if (shouldProxy(props)) { String userToProxy = props.getProperty("user.to.proxy"); if (securityEnabled) { String filelocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); _logger.info("Found token file " + filelocation); _logger.info("Security enabled is " + UserGroupInformation.isSecurityEnabled()); _logger.info("Setting mapreduce.job.credentials.binary to " + filelocation); System.setProperty("mapreduce.job.credentials.binary", filelocation); _logger.info("Proxying enabled."); loginUser = UserGroupInformation.getLoginUser(); _logger.info("Current logged in user is " + loginUser.getUserName()); proxyUser = UserGroupInformation.createProxyUser(userToProxy, loginUser); for (Token<?> token : loginUser.getTokens()) { proxyUser.addToken(token); } } else { proxyUser = UserGroupInformation.createRemoteUser(userToProxy); } _logger.info("Proxied as user " + userToProxy); } // Create the object using proxy if (shouldProxy(props)) { _javaObject = getObjectAsProxyUser(props, _logger, _jobName, className, proxyUser); } else { _javaObject = getObject(_jobName, className, props, _logger); } if (_javaObject == null) { _logger.info("Could not create java object to run job: " + className); throw new Exception("Could not create running object"); } _logger.info("Got object " + _javaObject.toString()); _cancelMethod = props.getProperty(CANCEL_METHOD_PARAM, DEFAULT_CANCEL_METHOD); final String runMethod = props.getProperty(RUN_METHOD_PARAM, DEFAULT_RUN_METHOD); _logger.info("Invoking method " + runMethod); if (shouldProxy(props)) { _logger.info("Proxying enabled."); runMethodAsUser(props, _javaObject, runMethod, proxyUser); } else { _logger.info("Proxy check failed, not proxying run."); runMethod(_javaObject, runMethod); } _isFinished = true; // Get the generated properties and store them to disk, to be read // by ProcessJob. try { final Method generatedPropertiesMethod = _javaObject.getClass() .getMethod(GET_GENERATED_PROPERTIES_METHOD, new Class<?>[] {}); Object outputGendProps = generatedPropertiesMethod.invoke(_javaObject, new Object[] {}); if (outputGendProps != null) { final Method toPropertiesMethod = outputGendProps.getClass().getMethod("toProperties", new Class<?>[] {}); Properties properties = (Properties) toPropertiesMethod.invoke(outputGendProps, new Object[] {}); Props outputProps = new Props(null, properties); outputGeneratedProperties(outputProps); } else { _logger.info(GET_GENERATED_PROPERTIES_METHOD + " method returned null. No properties to pass along"); } } catch (NoSuchMethodException e) { _logger.info(String.format( "Apparently there isn't a method[%s] on object[%s], using " + "empty Props object instead.", GET_GENERATED_PROPERTIES_METHOD, _javaObject)); outputGeneratedProperties(new Props()); } } catch (Exception e) { _isFinished = true; throw e; } }
From source file:azkaban.jobtype.HadoopSecureWrapperUtils.java
License:Apache License
/** * Perform all the magic required to get the proxyUser in a securitized grid * /*from w ww . jav a2 s . c o m*/ * @param userToProxy * @return a UserGroupInformation object for the specified userToProxy, which will also contain * the logged in user's tokens * @throws IOException */ private static UserGroupInformation createSecurityEnabledProxyUser(String userToProxy, String filelocation, Logger log) throws IOException { if (!new File(filelocation).exists()) { throw new RuntimeException("hadoop token file doesn't exist."); } log.info("Found token file. Setting " + HadoopSecurityManager.MAPREDUCE_JOB_CREDENTIALS_BINARY + " to " + filelocation); System.setProperty(HadoopSecurityManager.MAPREDUCE_JOB_CREDENTIALS_BINARY, filelocation); UserGroupInformation loginUser = null; loginUser = UserGroupInformation.getLoginUser(); log.info("Current logged in user is " + loginUser.getUserName()); UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(userToProxy, loginUser); for (Token<?> token : loginUser.getTokens()) { proxyUser.addToken(token); } return proxyUser; }
From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java
License:Apache License
@Override public synchronized void start() { final YarnConfiguration conf = new YarnConfiguration(getConfig()); final YarnRPC rpc = YarnRPC.create(conf); final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); UserGroupInformation currentUser; try {/*from www . j a va2 s. c o m*/ currentUser = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new YarnException(e); } if (UserGroupInformation.isSecurityEnabled()) { String tokenURLEncodedStr = System.getenv().get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME); Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>(); try { token.decodeFromUrlString(tokenURLEncodedStr); } catch (IOException e) { throw new YarnException(e); } SecurityUtil.setTokenService(token, rmAddress); if (LOG.isDebugEnabled()) { LOG.debug("AppMasterToken is " + token); } currentUser.addToken(token); } rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() { @Override public AMRMProtocol run() { return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf); } }); LOG.debug("Connecting to ResourceManager at " + rmAddress); super.start(); }
From source file:com.moz.fiji.schema.impl.hbase.HBaseFijiTable.java
License:Apache License
/** * Loads partitioned HFiles directly into the regions of this Fiji table. * * @param hfilePath Path of the HFiles to load. * @throws IOException on I/O error./*w w w .ja v a2 s. co m*/ */ public void bulkLoad(Path hfilePath) throws IOException { final LoadIncrementalHFiles loader = createHFileLoader(mConf); final String hFileScheme = hfilePath.toUri().getScheme(); Token<DelegationTokenIdentifier> hdfsDelegationToken = null; // If we're bulk loading from a secure HDFS, we should request and forward a delegation token. // LoadIncrementalHfiles will actually do this if none is provided, but because we call it // repeatedly in a short amount of time, this seems to trigger a possible race condition // where we ask to load the next HFile while there is a pending token cancellation request. // By requesting the token ourselves, it is re-used for each bulk load call. // Once we're done with the bulk loader we cancel the token. if (UserGroupInformation.isSecurityEnabled() && hFileScheme.equals(HDFS_SCHEME)) { final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); final DistributedFileSystem fileSystem = (DistributedFileSystem) hfilePath.getFileSystem(mConf); hdfsDelegationToken = fileSystem.getDelegationToken(RENEWER); ugi.addToken(hdfsDelegationToken); } try { // LoadIncrementalHFiles.doBulkLoad() requires an HTable instance, not an HTableInterface: final HTable htable = (HTable) mHTableFactory.create(mConf, mHBaseTableName); try { final List<Path> hfilePaths = Lists.newArrayList(); // Try to find any hfiles for partitions within the passed in path final FileStatus[] hfiles = hfilePath.getFileSystem(mConf).globStatus(new Path(hfilePath, "*")); for (FileStatus hfile : hfiles) { String partName = hfile.getPath().getName(); if (!partName.startsWith("_") && partName.endsWith(".hfile")) { Path partHFile = new Path(hfilePath, partName); hfilePaths.add(partHFile); } } if (hfilePaths.isEmpty()) { // If we didn't find any parts, add in the passed in parameter hfilePaths.add(hfilePath); } for (Path path : hfilePaths) { loader.doBulkLoad(path, htable); LOG.info("Successfully loaded: " + path.toString()); } } finally { htable.close(); } } catch (TableNotFoundException tnfe) { throw new InternalFijiError(tnfe); } // Cancel the HDFS delegation token if we requested one. if (null != hdfsDelegationToken) { try { hdfsDelegationToken.cancel(mConf); } catch (InterruptedException e) { LOG.warn("Failed to cancel HDFS delegation token.", e); } } }
From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java
License:Apache License
private void updateAMRMToken(Token token) throws IOException { org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>( token.getIdentifier().array(), token.getPassword().array(), new Text(token.getKind()), new Text(token.getService())); // Preserve the token service sent by the RM when adding the token // to ensure we replace the previous token setup by the RM. // Afterwards we can update the service address for the RPC layer. UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser(); currentUGI.addToken(amrmToken); amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig())); }
From source file:eu.stratosphere.yarn.ApplicationMaster.java
License:Apache License
public static void main(String[] args) throws Exception { final String yarnClientUsername = System.getenv(Client.ENV_CLIENT_USERNAME); LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting" + " user to execute Stratosphere ApplicationMaster/JobManager to '" + yarnClientUsername + "'"); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername); for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) { ugi.addToken(toks); }//from w ww . j av a 2 s . com ugi.doAs(new PrivilegedAction<Object>() { @Override public Object run() { try { new ApplicationMaster().run(); } catch (Exception e) { e.printStackTrace(); } return null; } }); }
From source file:eu.stratosphere.yarn.YarnTaskManagerRunner.java
License:Apache License
public static void main(final String[] args) throws IOException { Map<String, String> envs = System.getenv(); final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME); final String localDirs = envs.get(Environment.LOCAL_DIRS.key()); // configure local directory final String[] newArgs = Arrays.copyOf(args, args.length + 2); newArgs[newArgs.length - 2] = "-" + TaskManager.ARG_CONF_DIR; newArgs[newArgs.length - 1] = localDirs; LOG.info("Setting log path " + localDirs); LOG.info("YARN daemon runs as '" + UserGroupInformation.getCurrentUser().getShortUserName() + "' setting" + " user to execute Stratosphere TaskManager to '" + yarnClientUsername + "'"); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(yarnClientUsername); for (Token<? extends TokenIdentifier> toks : UserGroupInformation.getCurrentUser().getTokens()) { ugi.addToken(toks); }/*from www. ja va 2 s . c om*/ ugi.doAs(new PrivilegedAction<Object>() { @Override public Object run() { try { TaskManager.main(newArgs); } catch (Exception e) { e.printStackTrace(); } return null; } }); }
From source file:gobblin.util.ProxiedFileSystemUtils.java
License:Apache License
/** * Create a {@link FileSystem} that can perform any operations allowed the by the specified userNameToProxyAs. The * method first proxies as userNameToProxyAs, and then adds the specified {@link Token} to the given * {@link UserGroupInformation} object. It then uses the {@link UserGroupInformation#doAs(PrivilegedExceptionAction)} * method to create a {@link FileSystem}. * * @param userNameToProxyAs The name of the user the super user should proxy as * @param userNameToken The {@link Token} to add to the proxied user's {@link UserGroupInformation}. * @param fsURI The {@link URI} for the {@link FileSystem} that should be created * @param conf The {@link Configuration} for the {@link FileSystem} that should be created * * @return a {@link FileSystem} that can execute commands on behalf of the specified userNameToProxyAs *///from w ww . ja va 2 s.c o m static FileSystem createProxiedFileSystemUsingToken(@NonNull String userNameToProxyAs, @NonNull Token<?> userNameToken, URI fsURI, Configuration conf) throws IOException, InterruptedException { UserGroupInformation ugi = UserGroupInformation.createProxyUser(userNameToProxyAs, UserGroupInformation.getLoginUser()); ugi.addToken(userNameToken); return ugi.doAs(new ProxiedFileSystem(fsURI, conf)); }
From source file:gobblin.util.ProxiedFileSystemWrapper.java
License:Apache License
/** * Getter for proxiedFs, using the passed parameters to create an instance of a proxiedFs. * @param properties/* w ww . j av a 2s . c o m*/ * @param authType is either TOKEN or KEYTAB. * @param authPath is the KEYTAB location if the authType is KEYTAB; otherwise, it is the token file. * @param uri File system URI. * @throws IOException * @throws InterruptedException * @throws URISyntaxException * @return proxiedFs */ public FileSystem getProxiedFileSystem(State properties, AuthType authType, String authPath, String uri, final Configuration conf) throws IOException, InterruptedException, URISyntaxException { Preconditions.checkArgument( StringUtils.isNotBlank(properties.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME)), "State does not contain a proper proxy user name"); String proxyUserName = properties.getProp(ConfigurationKeys.FS_PROXY_AS_USER_NAME); UserGroupInformation proxyUser; switch (authType) { case KEYTAB: // If the authentication type is KEYTAB, log in a super user first before creating a proxy user. Preconditions.checkArgument( StringUtils .isNotBlank(properties.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS)), "State does not contain a proper proxy token file name"); String superUser = properties.getProp(ConfigurationKeys.SUPER_USER_NAME_TO_PROXY_AS_OTHERS); UserGroupInformation.loginUserFromKeytab(superUser, authPath); proxyUser = UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser()); break; case TOKEN: // If the authentication type is TOKEN, create a proxy user and then add the token to the user. proxyUser = UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser()); Optional<Token<?>> proxyToken = getTokenFromSeqFile(authPath, proxyUserName); if (proxyToken.isPresent()) { proxyUser.addToken(proxyToken.get()); } else { LOG.warn("No delegation token found for the current proxy user."); } break; default: LOG.warn( "Creating a proxy user without authentication, which could not perform File system operations."); proxyUser = UserGroupInformation.createProxyUser(proxyUserName, UserGroupInformation.getLoginUser()); break; } final URI fsURI = URI.create(uri); proxyUser.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws IOException { LOG.debug("Now performing file system operations as :" + UserGroupInformation.getCurrentUser()); proxiedFs = FileSystem.get(fsURI, conf); return null; } }); return this.proxiedFs; }