List of usage examples for org.apache.hadoop.yarn.api.records ApplicationAccessType VIEW_APP
ApplicationAccessType VIEW_APP
To view the source code for org.apache.hadoop.yarn.api.records ApplicationAccessType VIEW_APP.
Click Source Link
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 60000) public void testAppSubmissionAndNodeUpdate() throws Exception { MockRM rm = new MockRM(conf); rm.start();/*from w w w . j a v a 2 s . c o m*/ ClientRMService rmService = rm.getClientRMService(); GetApplicationsRequest getRequest = GetApplicationsRequest .newInstance(EnumSet.of(YarnApplicationState.KILLED)); ApplicationId appId1 = getApplicationId(100); ApplicationId appId2 = getApplicationId(101); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, null, null); try { rmService.submitApplication(submitRequest1); rmService.submitApplication(submitRequest2); } catch (YarnException e) { Assert.fail("Exception is not expected."); } assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size()); Thread.sleep(1000); //test persistance of schedulerapplication Map<String, SchedulerApplication> schedulerApplications = RMUtilities.getSchedulerApplications(); assertEquals("db does not contain good number of schedulerApplications", 2, schedulerApplications.size()); MockNM nm1 = rm.registerNode("host1:1234", 5120); MockNM nm2 = rm.registerNode("host2:5678", 10240); NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertEquals(4000, nodeHeartbeat.getNextHeartBeatInterval()); NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true); Assert.assertEquals(4000, nodeHeartbeat2.getNextHeartBeatInterval()); Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 30000) public void testForceKillApplication() throws Exception { MockRM rm = new MockRM(conf); rm.start();/*from w w w . j a v a 2 s . c om*/ ClientRMService rmService = rm.getClientRMService(); GetApplicationsRequest getRequest = GetApplicationsRequest .newInstance(EnumSet.of(YarnApplicationState.KILLED)); ApplicationId appId1 = getApplicationId(100); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null); try { rmService.submitApplication(submitRequest1); } catch (YarnException e) { Assert.fail("Exception is not expected."); } assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size()); Thread.sleep(1000); //TODO: check what have to be present in the db Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }
From source file:org.apache.apex.engine.security.ACLManager.java
License:Apache License
public static void setupUserACLs(ContainerLaunchContext launchContext, String userName, Configuration conf) throws IOException { logger.debug("Setup login acls {}", userName); if (areACLsRequired(conf)) { logger.debug("Configuring ACLs for {}", userName); Map<ApplicationAccessType, String> acls = Maps.newHashMap(); acls.put(ApplicationAccessType.VIEW_APP, userName); acls.put(ApplicationAccessType.MODIFY_APP, userName); launchContext.setApplicationACLs(acls); }/*w ww. j a v a 2 s . c om*/ }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncher.java
License:Apache License
/** * Setup and submit the Gobblin Yarn application. * * @throws IOException if there's anything wrong setting up and submitting the Yarn application * @throws YarnException if there's anything wrong setting up and submitting the Yarn application *//*ww w . j a v a 2 s.c om*/ @VisibleForTesting ApplicationId setupAndSubmitApplication() throws IOException, YarnException { YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); ApplicationId applicationId = appSubmissionContext.getApplicationId(); GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse(); // Set up resource type requirements for ApplicationMaster Resource resource = prepareContainerResource(newApplicationResponse); // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId); ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext .setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory()))); Map<ApplicationAccessType, String> acls = new HashMap<>(1); acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl); amContainerLaunchContext.setApplicationACLs(acls); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); } // Setup the application submission context appSubmissionContext.setApplicationName(this.applicationName); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext); // Also setup container local resources by copying local jars and files the container need to HDFS addContainerLocalResources(applicationId); // Submit the application LOGGER.info("Submitting application " + applicationId); this.yarnClient.submitApplication(appSubmissionContext); LOGGER.info("Application successfully submitted and accepted"); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue()); return applicationId; }
From source file:org.apache.gobblin.yarn.YarnService.java
License:Apache License
protected ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); Map<String, LocalResource> resourceMap = Maps.newHashMap(); addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);/*from w ww .java 2 s . c o m*/ addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap); addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) { addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap); } ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(resourceMap); containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName))); Map<ApplicationAccessType, String> acls = new HashMap<>(1); acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl); containerLaunchContext.setApplicationACLs(acls); if (UserGroupInformation.isSecurityEnabled()) { containerLaunchContext.setTokens(this.tokens.duplicate()); } return containerLaunchContext; }
From source file:org.apache.oozie.action.hadoop.TestYarnACLHandler.java
License:Apache License
private void assertViewACLset() { verifyACLset("View ACL", VIEW_ACL_VALUE, ApplicationAccessType.VIEW_APP); }
From source file:org.apache.oozie.action.hadoop.YarnACLHandler.java
License:Apache License
public void setACLs(ContainerLaunchContext containerLaunchContext) { Map<ApplicationAccessType, String> aclDefinition = new HashMap<>(); String viewAcl = launcherConf.get(JavaActionExecutor.LAUNCER_VIEW_ACL); if (viewAcl != null) { LOG.info("Setting view-acl: [{0}]", viewAcl); aclDefinition.put(ApplicationAccessType.VIEW_APP, viewAcl); }/*from ww w .j a va 2 s . c o m*/ String modifyAcl = launcherConf.get(JavaActionExecutor.LAUNCER_MODIFY_ACL); if (modifyAcl != null) { LOG.info("Setting modify-acl: [{0}]", modifyAcl); aclDefinition.put(ApplicationAccessType.MODIFY_APP, modifyAcl); } if (!aclDefinition.isEmpty()) { containerLaunchContext.setApplicationACLs(aclDefinition); } }
From source file:org.apache.samza.config.YarnConfig.java
License:Apache License
/** * Helper function to get all application acls * @return a map of {@link ApplicationAccessType} to {@link String} for all the acls defined *///from www. ja v a 2 s . c om public Map<ApplicationAccessType, String> getYarnApplicationAcls() { Map<ApplicationAccessType, String> acls = new HashMap<>(); String viewAcl = getYarnApplicationViewAcl(); String modifyAcl = getYarnApplicationModifyAcl(); if (viewAcl != null) { acls.put(ApplicationAccessType.VIEW_APP, viewAcl); } if (modifyAcl != null) { acls.put(ApplicationAccessType.MODIFY_APP, modifyAcl); } return acls; }
From source file:org.apache.tez.common.security.ACLManager.java
License:Apache License
public Map<ApplicationAccessType, String> toYARNACls() { Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2); if (!this.aclsEnabled) { acls.put(ApplicationAccessType.VIEW_APP, "*"); acls.put(ApplicationAccessType.MODIFY_APP, "*"); return acls; }/*from ww w . j a v a 2s .c om*/ acls.put(ApplicationAccessType.VIEW_APP, amUser); acls.put(ApplicationAccessType.MODIFY_APP, amUser); boolean viewAclsWildCard = false; boolean modifyAclsWildCard = false; if (users != null && !users.isEmpty()) { for (Entry<ACLType, Set<String>> entry : users.entrySet()) { if (entry.getKey().equals(ACLType.AM_VIEW_ACL)) { if (entry.getValue().contains(WILDCARD_ACL_VALUE)) { acls.put(ApplicationAccessType.VIEW_APP, "*"); viewAclsWildCard = true; continue; } else if (!entry.getValue().isEmpty()) { String aclsStr = acls.get(ApplicationAccessType.VIEW_APP); String commaSepList = toCommaSeparatedString(entry.getValue()); if (!commaSepList.isEmpty()) { aclsStr += "," + commaSepList; } acls.put(ApplicationAccessType.VIEW_APP, aclsStr); } } else if (entry.getKey().equals(ACLType.AM_MODIFY_ACL)) { if (entry.getValue().contains(WILDCARD_ACL_VALUE)) { acls.put(ApplicationAccessType.MODIFY_APP, "*"); modifyAclsWildCard = true; continue; } else if (!entry.getValue().isEmpty()) { String aclsStr = acls.get(ApplicationAccessType.MODIFY_APP); String commaSepList = toCommaSeparatedString(entry.getValue()); if (!commaSepList.isEmpty()) { aclsStr += "," + commaSepList; } acls.put(ApplicationAccessType.MODIFY_APP, aclsStr); } } } } if (groups != null && !groups.isEmpty()) { for (Entry<ACLType, Set<String>> entry : groups.entrySet()) { if (entry.getKey().equals(ACLType.AM_VIEW_ACL) && !viewAclsWildCard && !entry.getValue().isEmpty()) { // Append groups only if wild card not set String aclsStr = acls.containsKey(ApplicationAccessType.VIEW_APP) ? acls.get(ApplicationAccessType.VIEW_APP) : ""; aclsStr += " " + toCommaSeparatedString(entry.getValue()); acls.put(ApplicationAccessType.VIEW_APP, aclsStr); } else if (entry.getKey().equals(ACLType.AM_MODIFY_ACL) && !modifyAclsWildCard && !entry.getValue().isEmpty()) { // Append groups only if wild card not set String aclsStr = acls.containsKey(ApplicationAccessType.MODIFY_APP) ? acls.get(ApplicationAccessType.MODIFY_APP) : ""; aclsStr += " " + toCommaSeparatedString(entry.getValue()); acls.put(ApplicationAccessType.MODIFY_APP, aclsStr); } } } return acls; }
From source file:org.apache.tez.common.security.TestACLManager.java
License:Apache License
@Test(timeout = 5000) public void testConvertToYARNACLs() { String currentUser = "c1"; Configuration conf = new Configuration(false); String viewACLs = "user1,user4,, grp3,grp4 "; conf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewACLs); conf.set(TezConfiguration.TEZ_AM_MODIFY_ACLS, " * "); ACLManager aclManager = new ACLManager(currentUser, conf); Map<ApplicationAccessType, String> yarnAcls = aclManager.toYARNACls(); Assert.assertTrue(yarnAcls.containsKey(ApplicationAccessType.VIEW_APP)); Assert.assertEquals("c1,user1,user4 grp3,grp4", yarnAcls.get(ApplicationAccessType.VIEW_APP)); Assert.assertTrue(yarnAcls.containsKey(ApplicationAccessType.MODIFY_APP)); Assert.assertEquals("*", yarnAcls.get(ApplicationAccessType.MODIFY_APP)); viewACLs = " grp3,grp4 "; conf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewACLs); ACLManager aclManager1 = new ACLManager(currentUser, conf); yarnAcls = aclManager1.toYARNACls(); Assert.assertEquals("c1 grp3,grp4", yarnAcls.get(ApplicationAccessType.VIEW_APP)); }