List of usage examples for org.apache.hadoop.security UserGroupInformation getCurrentUser
@InterfaceAudience.Public @InterfaceStability.Evolving public static UserGroupInformation getCurrentUser() throws IOException
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 30000) public void testForceKillApplication() throws Exception { MockRM rm = new MockRM(conf); rm.start();//from w ww . ja va 2s . c o m ClientRMService rmService = rm.getClientRMService(); GetApplicationsRequest getRequest = GetApplicationsRequest .newInstance(EnumSet.of(YarnApplicationState.KILLED)); ApplicationId appId1 = getApplicationId(100); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null); try { rmService.submitApplication(submitRequest1); } catch (YarnException e) { Assert.fail("Exception is not expected."); } assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size()); Thread.sleep(1000); //TODO: check what have to be present in the db Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }
From source file:io.hops.tensorflow.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException// ww w . ja v a 2s .c o m * @throws IOException */ public void run() throws YarnException, IOException, InterruptedException { LOG.info("Starting ApplicationMaster. " + "Workers: " + numWorkers + ", Parameter servers: " + numPses); clusterSpecServer = new ClusterSpecGeneratorServer(appAttemptID.getApplicationId().toString(), numTotalContainers, numWorkers); LOG.info("Starting ClusterSpecGeneratorServer"); int port = 2222; while (true) { try { clusterSpecServer.start(port); break; } catch (IOException e) { port++; } } environment.put("YARNTF_AM_ADDRESS", InetAddress.getLocalHost().getHostName() + ":" + port); environment.put("YARNTF_APPLICATION_ID", appAttemptID.getApplicationId().toString()); // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class // are marked as LimitedPrivate Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); rmWrapper = new RMWrapper(this); rmWrapper.getClient().init(conf); rmWrapper.getClient().start(); nmWrapper = new NMWrapper(this); nmWrapper.getClient().init(conf); nmWrapper.getClient().start(); timelineHandler = new TimelineHandler(appAttemptID.toString(), domainId, appSubmitterUgi); timelineHandler.startClient(conf); if (timelineHandler.isClientNotNull()) { timelineHandler.publishApplicationAttemptEvent(YarntfEvent.YARNTF_APP_ATTEMPT_START); } // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); appMasterTrackingUrl = InetAddress.getLocalHost().getHostName() + ":" + TensorBoardServer.spawn(this); RegisterApplicationMasterResponse response = rmWrapper.getClient() .registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); int maxGPUS = response.getMaximumResourceCapability().getGPUs(); LOG.info("Max gpus capabililty of resources in this cluster " + maxGPUS); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } if (containerGPUs > maxGPUS) { LOG.info("Container gpus specified above max threshold of cluster." + " Using max value." + ", specified=" + containerGPUs + ", max=" + maxGPUS); containerGPUs = maxGPUS; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); // Stop eventual containers from previous attempts for (Container prevContainer : previousAMRunningContainers) { LOG.info("Releasing YARN container " + prevContainer.getId()); rmWrapper.getClient().releaseAssignedContainer(prevContainer.getId()); } // Send request for containers to RM for (int i = 0; i < numWorkers; i++) { ContainerRequest workerRequest = setupContainerAskForRM(true); rmWrapper.getClient().addContainerRequest(workerRequest); } numRequestedContainers.addAndGet(numWorkers); for (int i = 0; i < numPses; i++) { ContainerRequest psRequest = setupContainerAskForRM(false); rmWrapper.getClient().addContainerRequest(psRequest); } numRequestedContainers.addAndGet(numPses); }
From source file:io.hops.util.GroupMembershipService.java
License:Apache License
@Override public synchronized void serviceInit(Configuration conf) throws Exception { this.conf = conf; groupMembershipServiceAddress = conf.getSocketAddr(YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_GROUP_MEMBERSHIP_ADDRESS, YarnConfiguration.DEFAULT_RM_GROUP_MEMBERSHIP_ADDRESS, YarnConfiguration.DEFAULT_RM_GROUP_MEMBERSHIP_PORT); adminAcl = new AccessControlList( conf.get(YarnConfiguration.YARN_ADMIN_ACL, YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)); if (HAUtil.isHAEnabled(conf)) { this.rmId = HAUtil.getRMHAId(conf); }/*from w w w . j av a 2s . co m*/ daemonUser = UserGroupInformation.getCurrentUser(); authorizer = YarnAuthorizationProvider.getInstance(conf); authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation.getCurrentUser()); LOG.info("init groupMembershipService " + this.rmId); }
From source file:io.hops.util.GroupMembershipService.java
License:Apache License
private RefreshAdminAclsResponse refreshAdminAcls(boolean checkRMHAState) throws YarnException, IOException { String argName = "refreshAdminAcls"; UserGroupInformation user = checkAcls(argName); if (checkRMHAState) { checkRMStatus(user.getShortUserName(), argName, "refresh Admin ACLs."); }//from ww w . j a va 2 s . com Configuration conf = getConfiguration(new Configuration(false), YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation.getCurrentUser()); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance(RefreshAdminAclsResponse.class); }
From source file:joshelser.Client.java
License:Apache License
public static void main(String[] args) throws Exception { Opts opts = new Opts(); // Parse the options opts.parseArgs(Client.class, args); // Open up a socket to the server:port TTransport transport = new TSocket(opts.server, opts.port); Map<String, String> saslProperties = new HashMap<String, String>(); // Use authorization and confidentiality saslProperties.put(Sasl.QOP, "auth-conf"); log.info("Security is enabled: {}", UserGroupInformation.isSecurityEnabled()); // Log in via UGI, ensures we have logged in with our KRB credentials UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); log.info("Current user: {}", currentUser); // SASL client transport -- does the Kerberos lifting for us TSaslClientTransport saslTransport = new TSaslClientTransport("GSSAPI", // tell SASL to use GSSAPI, which supports Kerberos null, // authorizationid - null opts.primary, // kerberos primary for server - "myprincipal" in myprincipal/my.server.com@MY.REALM opts.instance, // kerberos instance for server - "my.server.com" in myprincipal/my.server.com@MY.REALM saslProperties, // Properties set, above null, // callback handler - null transport); // underlying transport // Make sure the transport is opened as the user we logged in as TUGIAssumingTransport ugiTransport = new TUGIAssumingTransport(saslTransport, currentUser); // Setup our thrift client to our custom thrift service HdfsService.Client client = new HdfsService.Client(new TBinaryProtocol(ugiTransport)); // Open the transport ugiTransport.open();/*from w w w . ja v a2 s.c om*/ // Invoke the RPC String response = client.ls(opts.dir); // Print out the result System.out.println("$ ls " + opts.dir + "\n" + response); // Close the transport (don't leak resources) transport.close(); }
From source file:joshelser.HdfsServiceImpl.java
License:Apache License
@Override public String ls(String directory) throws TException { StringBuilder sb = new StringBuilder(64); try {/* ww w .j av a2 s. com*/ log.debug("Running as {}", UserGroupInformation.getCurrentUser()); for (FileStatus stat : fs.listStatus(new Path(directory))) { sb.append(stat.getPath().getName()); if (stat.isDirectory()) { sb.append("/"); } sb.append("\n"); } } catch (FileNotFoundException e) { System.err.println("Got FileNotFoundException"); e.printStackTrace(System.err); throw new TException(e); } catch (IllegalArgumentException e) { System.err.println("Got IllegalArgumentException"); e.printStackTrace(System.err); throw new TException(e); } catch (IOException e) { System.err.println("Got IOException"); e.printStackTrace(System.err); throw new TException(e); } return sb.toString(); }
From source file:me.haosdent.noya.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws org.apache.hadoop.yarn.exceptions.YarnException * @throws java.io.IOException/*www .j a v a 2s. com*/ */ @SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException { LOG.info("Starting ApplicationMaster"); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Setup local RPC Server to accept status requests directly from clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info("Received " + previousAMRunningContainers.size() + " previous AM's running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainersToRequest); try { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END); } catch (Exception e) { LOG.error("App Attempt start event coud not be pulished for " + appAttemptID.toString(), e); } }
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * Populate allTokens with the tokens received */// www . ja va 2 s .co m private void getAllTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (LOG.isDebugEnabled()) { LOG.debug("Token type : {}", token.getKind()); } if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } this.allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); }
From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java
License:Apache License
/** * Application entry point/*w w w .jav a 2s. com*/ * * @param args * command-line args (set by GuaguaYarnClient, if any) */ public static void main(final String[] args) { LOG.info("Starting GuaguaAppMaster. "); String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); try { UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appMasterUgi = UserGroupInformation.createRemoteUser(jobUserName); appMasterUgi.addCredentials(credentials); // Now remove the AM->RM token so tasks don't have it Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } final GuaguaAppMaster appMaster = new GuaguaAppMaster(containerId, appAttemptId, conf); appMasterUgi.doAs(new PrivilegedAction<Void>() { @Override public Void run() { boolean result = false; try { result = appMaster.run(); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } if (result) { LOG.info("Guagua Application Master completed successfully. exiting"); System.exit(0); } else { LOG.info("Guagua Application Master failed. exiting"); System.exit(2); } return null; } }); } catch (Throwable t) { LOG.error("GuaguaAppMaster caught a top-level exception in main.", t); System.exit(1); } }
From source file:ml.shifu.guagua.yarn.GuaguaYarnTask.java
License:Apache License
public static void main(String[] args) { LOG.info("args:{}", Arrays.toString(args)); if (args.length != 7) { throw new IllegalStateException(String.format( "GuaguaYarnTask could not construct a TaskAttemptID for the Guagua job from args: %s", Arrays.toString(args))); }/*w ww. j a v a 2 s. c o m*/ String containerIdString = System.getenv().get(Environment.CONTAINER_ID.name()); if (containerIdString == null) { // container id should always be set in the env by the framework throw new IllegalArgumentException("ContainerId not found in env vars."); } ContainerId containerId = ConverterUtils.toContainerId(containerIdString); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); try { Configuration conf = new YarnConfiguration(); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); conf.set(MRJobConfig.USER_NAME, jobUserName); UserGroupInformation.setConfiguration(conf); // Security framework already loaded the tokens into current UGI, just use them Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); LOG.info("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.info(token.toString()); } UserGroupInformation appTaskUGI = UserGroupInformation.createRemoteUser(jobUserName); appTaskUGI.addCredentials(credentials); @SuppressWarnings("rawtypes") final GuaguaYarnTask<?, ?> guaguaYarnTask = new GuaguaYarnTask(appAttemptId, containerId, Integer.parseInt(args[args.length - 3]), args[args.length - 2], args[args.length - 1], conf); appTaskUGI.doAs(new PrivilegedAction<Void>() { @Override public Void run() { guaguaYarnTask.run(); return null; } }); } catch (Throwable t) { LOG.error("GuaguaYarnTask threw a top-level exception, failing task", t); System.exit(2); } System.exit(0); }