List of usage examples for org.apache.hadoop.security UserGroupInformation addCredentials
public void addCredentials(Credentials credentials)
From source file:co.cask.cdap.security.impersonation.RemoteUGIProvider.java
License:Apache License
@Override protected UserGroupInformation createUGI(ImpersonationInfo impersonationInfo) throws IOException { String credentialsURI = executeRequest(impersonationInfo).getResponseBodyAsString(); LOG.debug("Received response: {}", credentialsURI); Location location = locationFactory.create(URI.create(credentialsURI)); try {/*from w ww. j a v a 2 s . com*/ UserGroupInformation impersonatedUGI = UserGroupInformation .createRemoteUser(impersonationInfo.getPrincipal()); impersonatedUGI.addCredentials(readCredentials(location)); return impersonatedUGI; } finally { try { if (!location.delete()) { LOG.warn("Failed to delete location: {}", location); } } catch (IOException e) { LOG.warn("Exception raised when deleting location {}", location, e); } } }
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * Application entry point/*from w ww . j av a 2 s . c o m*/ * * @param args * command-line args (set by GuaguaYarnClient, if any) */ public static void main(final String[] args) { LOG.info("Starting GuaguaAppMaster. "); String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); try { UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appMasterUgi = UserGroupInformation.createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); // Now remove the AM->RM token so tasks don't have it Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } final GuaguaAppMaster appMaster = new GuaguaAppMaster(containerId, appAttemptId, conf); appMasterUgi.doAs(new PrivilegedAction<Void>() { @Override public Void run() { boolean result = false; try { result = appMaster.run(); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } if (result) { LOG.info("Guagua Application Master completed successfully. exiting"); System.exit(0); } else { LOG.info("Guagua Application Master failed. exiting"); System.exit(2); } return null; } }); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } }
From source file:ml.shifu.guagua.yarn.GuaguaYarnTask.java
License:Apache License
public static void main(String[] args) { LOG.info("args:{}", Arrays.toString(args)); if (args.length != 7) { throw new IllegalStateException(String.format( "GuaguaYarnTask could not construct a TaskAttemptID for the Guagua job from args: %s", Arrays.toString(args))); }/* w w w. j av a 2 s.c o m*/ String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); try { Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appTaskUGI = UserGroupInformation.createRemoteUser(jobUserName); appTaskUGI.addCredentials(credentials); @SuppressWarnings("rawtypes") final GuaguaYarnTask<?, ?> guaguaYarnTask = new GuaguaYarnTask(appAttemptId, containerId, Integer.parseInt(args[args.length - 3]), args[args.length - 2], args[args.length - 1], conf); appTaskUGI.doAs(new PrivilegedAction<Void>() { @Override public Void run() { guaguaYarnTask.run(); return null; } }); } catch (Throwable t) { LOG.error("GuaguaYarnTask threw a top-level exception, failing task", t); System.exit(2); } System.exit(0); }
From source file:org.apache.falcon.catalog.HiveCatalogService.java
License:Apache License
/** * This is used from with in an oozie job. * * @param conf conf object//www. j a v a 2 s.c om * @param metastoreUrl metastore uri * @return hive metastore client handle * @throws FalconException */ private static HiveMetaStoreClient createClient(Configuration conf, String metastoreUrl) throws FalconException { try { LOG.info("Creating HCatalog client object for metastore {} using conf {}", metastoreUrl, conf.toString()); final Credentials credentials = getCredentials(conf); Configuration jobConf = credentials != null ? copyCredentialsToConf(conf, credentials) : conf; HiveConf hcatConf = createHiveConf(jobConf, metastoreUrl); if (UserGroupInformation.isSecurityEnabled()) { hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, conf.get(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname)); hcatConf.set(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, "true"); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); ugi.addCredentials(credentials); // credentials cannot be null } return new HiveMetaStoreClient(hcatConf); } catch (Exception e) { throw new FalconException("Exception creating HiveMetaStoreClient: " + e.getMessage(), e); } }
From source file:org.apache.metron.maas.service.yarn.YarnUtils.java
License:Apache License
public UserGroupInformation createUserGroup(Credentials credentials) throws IOException { credentials = credentials == null ? UserGroupInformation.getCurrentUser().getCredentials() : credentials; String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); UserGroupInformation appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); return appSubmitterUgi; }
From source file:org.apache.oozie.action.hadoop.LauncherAM.java
License:Apache License
private static UserGroupInformation getUserGroupInformation(Configuration launcherConf, Text... kindToFilter) throws IOException { final String submitterUser = launcherConf.get(OOZIE_SUBMITTER_USER); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); filterTokensByKind(credentials, kindToFilter); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(submitterUser); ugi.addCredentials(credentials); return ugi;/*from ww w.j a va 2 s . co m*/ }
From source file:org.apache.tez.runtime.task.TezChild.java
License:Apache License
/** * Setup// ww w. j a v a 2 s. co m * * @param containerTask * the new task specification. Must be a valid task * @param childUGI * the old UGI instance being used * @return childUGI */ UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask, UserGroupInformation childUGI) { // Re-use the UGI only if the Credentials have not changed. Preconditions.checkState(!containerTask.shouldDie()); Preconditions.checkState(containerTask.getTaskSpec() != null); if (containerTask.haveCredentialsChanged()) { LOG.info("Refreshing UGI since Credentials have changed"); Credentials taskCreds = containerTask.getCredentials(); if (taskCreds != null) { LOG.info("Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys=" + taskCreds.numberOfSecretKeys()); childUGI = UserGroupInformation.createRemoteUser(user); childUGI.addCredentials(containerTask.getCredentials()); } else { LOG.info("Not loading any credentials, since no credentials provided"); } } return childUGI; }
From source file:x10.x10rt.yarn.ApplicationMaster.java
License:Open Source License
private void setup() throws IOException, YarnException { LOG.info("Starting ApplicationMaster"); // Remove the AM->RM token so that containers cannot access it. Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token);// ww w . j av a 2 s . com if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); UserGroupInformation appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, new RMCallbackHandler()); resourceManager.init(conf); resourceManager.start(); nodeManager = new NMClientAsyncImpl(new NMCallbackHandler(this)); nodeManager.init(conf); nodeManager.start(); // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); { int slash = appMasterHostname.indexOf('/'); if (slash != -1) appMasterHostname = appMasterHostname.substring(0, slash); } // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. // TODO: should we reject instead of modifying to fit? if (memoryPerPlaceInMb > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + memoryPerPlaceInMb + ", max=" + maxMem); memoryPerPlaceInMb = maxMem; } if (coresPerPlace > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + coresPerPlace + ", max=" + maxVCores); coresPerPlace = maxVCores; } else if (coresPerPlace == 0) { LOG.info("Container virtual cores specified as auto (X10_NTHREADS=0)." + " Using max value." + ", specified=" + coresPerPlace + ", max=" + maxVCores); coresPerPlace = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = initialNumPlaces - previousAMRunningContainers.size(); // open a local port for X10rt management, and register it with the selector launcherChannel = ServerSocketChannel.open(); //launcherChannel.bind(new InetSocketAddress(appMasterHostname, 0)); // bind to the visible network hostname and random port launcherChannel.bind(null); launcherChannel.configureBlocking(false); appMasterPort = launcherChannel.socket().getLocalPort(); launcherChannel.register(selector, SelectionKey.OP_ACCEPT); numRequestedContainers.set(initialNumPlaces); // Send request for containers to RM for (int i = 0; i < numTotalContainersToRequest; ++i) { Resource capability = Resource.newInstance(memoryPerPlaceInMb, coresPerPlace); ContainerRequest request = new ContainerRequest(capability, null, null, Priority.newInstance(0)); LOG.info("Requested container ask: " + request.toString()); resourceManager.addContainerRequest(request); pendingRequests.add(request); } }
From source file:yarnkit.appmaster.ApplicationMasterService.java
License:Apache License
@Override protected void startUp() throws Exception { LOG.info("Starting Application Master"); // create security tokens Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); ByteBuffer securityTokens = YarnUtils.getSecurityToken(credentials); // Create appSubmitterUgi and add original tokens to it String userName = System.getenv(ApplicationConstants.Environment.USER.name()); UserGroupInformation appSubmitterUgi = UserGroupInformation.createRemoteUser(userName); // remove the AM->RM token so that containers cannot access it. YarnUtils.removeToken(credentials, AMRMTokenIdentifier.KIND_NAME); appSubmitterUgi.addCredentials(credentials); // start a resource manager (RM) this.resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, this); resourceManager.init(yarnConf);/*from w w w. java 2s .c om*/ resourceManager.start(); // register a application master (AM) to resource manager (RM) final RegisterApplicationMasterResponse registration; try { registration = resourceManager.registerApplicationMaster(parameters.getHostname(), parameters.getClientPort(), parameters.getTrackingUrl()); LOG.info("Registered Application Master: " + registration); } catch (Exception e) { LOG.error("Exception thrown registering Application Master", e); stop(); return; } // assign containers ContainerLaunchContextFactory factory = new ContainerLaunchContextFactory( registration.getMaximumResourceCapability(), securityTokens); ContainerLaunchParameters containerLaunchParams = parameters.getContainerLaunchParameters(); this.tracker = new ContainerTracker(this, containerLaunchParams); tracker.init(factory, yarnConf); this.hasRunningContainers = true; }