List of usage examples for org.apache.hadoop.yarn.api.records ApplicationId toString
@Override
public String toString()
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Create an ApplicationSubmissionContext to launch a Tez AM * @param appId Application Id//from w w w .j av a 2s.c o m * @param dag DAG to be submitted * @param amName Name for the application * @param amConfig AM Configuration * @param tezJarResources Resources to be used by the AM * @param sessionCreds the credential object which will be populated with session specific * @param historyACLPolicyManager * @return an ApplicationSubmissionContext to launch a Tez AM * @throws IOException * @throws YarnException */ @Private @VisibleForTesting public static ApplicationSubmissionContext createApplicationSubmissionContext(ApplicationId appId, DAG dag, String amName, AMConfiguration amConfig, Map<String, LocalResource> tezJarResources, Credentials sessionCreds, boolean tezLrsAsArchive, TezApiVersionInfo apiVersionInfo, HistoryACLPolicyManager historyACLPolicyManager) throws IOException, YarnException { Preconditions.checkNotNull(sessionCreds); TezConfiguration conf = amConfig.getTezConfiguration(); FileSystem fs = TezClientUtils.ensureStagingDirExists(conf, TezCommonUtils.getTezBaseStagingPath(conf)); String strAppId = appId.toString(); Path tezSysStagingPath = TezCommonUtils.createTezSystemStagingPath(conf, strAppId); Path binaryConfPath = TezCommonUtils.getTezConfStagingPath(tezSysStagingPath); binaryConfPath = fs.makeQualified(binaryConfPath); // Setup resource requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT)); capability.setVirtualCores(amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT)); if (LOG.isDebugEnabled()) { LOG.debug("AppMaster capability = " + capability); } // Setup required Credentials for the AM launch. DAG specific credentials // are handled separately. ByteBuffer securityTokens = null; // Setup security tokens Credentials amLaunchCredentials = new Credentials(); if (amConfig.getCredentials() != null) { amLaunchCredentials.addAll(amConfig.getCredentials()); } // Add Staging dir creds to the list of session credentials. TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { binaryConfPath }, conf); // Add session specific credentials to the AM credentials. amLaunchCredentials.mergeAll(sessionCreds); DataOutputBuffer dob = new DataOutputBuffer(); amLaunchCredentials.writeTokenStorageToStream(dob); securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Setup the command to run the AM List<String> vargs = new ArrayList<String>(8); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability); vargs.add(amOpts); String amLogLevelString = amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_LOG_LEVEL, TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT); String[] amLogParams = parseLogParams(amLogLevelString); String amLogLevel = amLogParams[0]; maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs); // FIX sun bug mentioned in TEZ-327 vargs.add("-Dsun.nio.ch.bugLevel=''"); vargs.add(TezConstants.TEZ_APPLICATION_MASTER_CLASS); if (dag == null) { vargs.add("--" + TezConstants.TEZ_SESSION_MODE_CLI_OPTION); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDOUT); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR); Vector<String> vargsFinal = new Vector<String>(8); // Final command StringBuilder mergedCommand = new StringBuilder(); for (CharSequence str : vargs) { mergedCommand.append(str).append(" "); } vargsFinal.add(mergedCommand.toString()); if (LOG.isDebugEnabled()) { LOG.debug("Command to launch container for ApplicationMaster is : " + mergedCommand); } Map<String, String> environment = new TreeMap<String, String>(); TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV, TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); addVersionInfoToEnv(environment, apiVersionInfo); addLogParamsToEnv(environment, amLogParams); Map<String, LocalResource> amLocalResources = new TreeMap<String, LocalResource>(); // Not fetching credentials for AMLocalResources. Expect this to be provided via AMCredentials. if (amConfig.getAMLocalResources() != null) { amLocalResources.putAll(amConfig.getAMLocalResources()); } amLocalResources.putAll(tezJarResources); // Setup Session ACLs and update conf as needed Map<String, String> aclConfigs = null; if (historyACLPolicyManager != null) { if (dag == null) { aclConfigs = historyACLPolicyManager.setupSessionACLs(amConfig.getTezConfiguration(), appId); } else { // Non-session mode // As only a single DAG is support, we should combine AM and DAG ACLs under the same // acl management layer aclConfigs = historyACLPolicyManager.setupNonSessionACLs(amConfig.getTezConfiguration(), appId, dag.getDagAccessControls()); } } // emit conf as PB file ConfigurationProto finalConfProto = createFinalConfProtoForApp(amConfig.getTezConfiguration(), aclConfigs); FSDataOutputStream amConfPBOutBinaryStream = null; try { amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath); finalConfProto.writeTo(amConfPBOutBinaryStream); } finally { if (amConfPBOutBinaryStream != null) { amConfPBOutBinaryStream.close(); } } LocalResource binaryConfLRsrc = TezClientUtils.createLocalResource(fs, binaryConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); amConfig.setBinaryConfLR(binaryConfLRsrc); amLocalResources.put(TezConstants.TEZ_PB_BINARY_CONF_NAME, binaryConfLRsrc); // Create Session Jars definition to be sent to AM as a local resource Path sessionJarsPath = TezCommonUtils.getTezAMJarStagingPath(tezSysStagingPath); FSDataOutputStream sessionJarsPBOutStream = null; try { sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, sessionJarsPath); // Write out the initial list of resources which will be available in the AM DAGProtos.PlanLocalResourcesProto amResourceProto; if (amLocalResources != null && !amLocalResources.isEmpty()) { amResourceProto = DagTypeConverters.convertFromLocalResources(amLocalResources); } else { amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance(); } amResourceProto.writeDelimitedTo(sessionJarsPBOutStream); } finally { if (sessionJarsPBOutStream != null) { sessionJarsPBOutStream.close(); } } LocalResource sessionJarsPBLRsrc = TezClientUtils.createLocalResource(fs, sessionJarsPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); amLocalResources.put(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, sessionJarsPBLRsrc); String user = UserGroupInformation.getCurrentUser().getShortUserName(); ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration()); Map<ApplicationAccessType, String> acls = aclManager.toYARNACls(); if (dag != null) { DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, sessionCreds); // emit protobuf DAG file style Path binaryPath = TezCommonUtils.getTezBinPlanStagingPath(tezSysStagingPath); if (LOG.isDebugEnabled()) { LOG.debug("Stage directory information for AppId :" + appId + " tezSysStagingPath :" + tezSysStagingPath + " binaryConfPath :" + binaryConfPath + " sessionJarsPath :" + sessionJarsPath + " binaryPlanPath :" + binaryPath); } FSDataOutputStream dagPBOutBinaryStream = null; try { //binary output dagPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryPath); dagPB.writeTo(dagPBOutBinaryStream); } finally { if (dagPBOutBinaryStream != null) { dagPBOutBinaryStream.close(); } } amLocalResources.put(TezConstants.TEZ_PB_PLAN_BINARY_NAME, TezClientUtils.createLocalResource(fs, binaryPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) { Path textPath = localizeDagPlanAsText(dagPB, fs, amConfig, strAppId, tezSysStagingPath); amLocalResources.put(TezConstants.TEZ_PB_PLAN_TEXT_NAME, TezClientUtils.createLocalResource(fs, textPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); } } // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(amLocalResources, environment, vargsFinal, null, securityTokens, acls); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE); appContext.setApplicationId(appId); appContext.setResource(capability); if (amConfig.getQueueName() != null) { appContext.setQueue(amConfig.getQueueName()); } appContext.setApplicationName(amName); appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION, TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT)); appContext.setAMContainerSpec(amContainer); appContext.setMaxAppAttempts(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT)); return appContext; }
From source file:org.apache.tez.dag.app.TestDAGAppMaster.java
License:Apache License
@SuppressWarnings("deprecation") private void testDagCredentials(boolean doMerge) throws IOException { TezConfiguration conf = new TezConfiguration(); conf.setBoolean(TezConfiguration.TEZ_AM_CREDENTIALS_MERGE, doMerge); conf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true); conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, TEST_DIR.toString()); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1); // create some sample AM credentials Credentials amCreds = new Credentials(); JobTokenSecretManager jtsm = new JobTokenSecretManager(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appId.toString())); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, jtsm); sessionToken.setService(identifier.getJobId()); TokenCache.setSessionToken(sessionToken, amCreds); TestTokenSecretManager ttsm = new TestTokenSecretManager(); Text tokenAlias1 = new Text("alias1"); Token<TestTokenIdentifier> amToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken1")), ttsm); amCreds.addToken(tokenAlias1, amToken1); Text tokenAlias2 = new Text("alias2"); Token<TestTokenIdentifier> amToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("amtoken2")), ttsm); amCreds.addToken(tokenAlias2, amToken2); FileSystem fs = FileSystem.getLocal(conf); FSDataOutputStream sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, new Path(TEST_DIR.toString(), TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)); DAGProtos.PlanLocalResourcesProto.getDefaultInstance().writeDelimitedTo(sessionJarsPBOutStream); sessionJarsPBOutStream.close();// w w w . ja v a2 s .com DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, TEST_DIR.toString(), new String[] { TEST_DIR.toString() }, new String[] { TEST_DIR.toString() }, new TezApiVersionInfo().getVersion(), 1, amCreds, "someuser", null); am.init(conf); am.start(); // create some sample DAG credentials Credentials dagCreds = new Credentials(); Token<TestTokenIdentifier> dagToken1 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken1")), ttsm); dagCreds.addToken(tokenAlias2, dagToken1); Text tokenAlias3 = new Text("alias3"); Token<TestTokenIdentifier> dagToken2 = new Token<TestTokenIdentifier>( new TestTokenIdentifier(new Text("dagtoken2")), ttsm); dagCreds.addToken(tokenAlias3, dagToken2); TezDAGID dagId = TezDAGID.getInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("somedag") .setCredentialsBinary(DagTypeConverters.convertCredentialsToProto(dagCreds)).build(); DAGImpl dag = am.createDAG(dagPlan, dagId); Credentials fetchedDagCreds = dag.getCredentials(); am.stop(); Token<? extends TokenIdentifier> fetchedToken1 = fetchedDagCreds.getToken(tokenAlias1); if (doMerge) { assertNotNull("AM creds missing from DAG creds", fetchedToken1); compareTestTokens(amToken1, fetchedDagCreds.getToken(tokenAlias1)); } else { assertNull("AM creds leaked to DAG creds", fetchedToken1); } compareTestTokens(dagToken1, fetchedDagCreds.getToken(tokenAlias2)); compareTestTokens(dagToken2, fetchedDagCreds.getToken(tokenAlias3)); }
From source file:org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.java
License:Apache License
private Map<String, String> createSessionDomain(Configuration tezConf, ApplicationId applicationId, DAGAccessControls dagAccessControls) throws IOException { String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID); if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) { if (domainId != null) { throw new TezUncheckedException( "ACLs disabled but DomainId is specified" + ", aclsEnabled=true, domainId=" + domainId); }/*from w w w.j a va 2 s . co m*/ return null; } boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE, TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT); if (domainId != null) { // do nothing LOG.info("Using specified domainId with Timeline, domainId=" + domainId); return null; } else { if (!autoCreateDomain) { // Error - Cannot fallback to default as that leaves ACLs open throw new TezUncheckedException( "Timeline DomainId is not specified and auto-create" + " Domains is disabled"); } domainId = DOMAIN_ID_PREFIX + applicationId.toString(); createTimelineDomain(domainId, tezConf, dagAccessControls); LOG.info("Created Timeline Domain for History ACLs, domainId=" + domainId); return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, domainId); } }
From source file:org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.java
License:Apache License
private Map<String, String> createDAGDomain(Configuration tezConf, ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) throws IOException { if (dagAccessControls == null) { // No DAG specific ACLs return null; }/*from w w w.j ava 2 s . c om*/ String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID); if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) { if (domainId != null) { throw new TezUncheckedException("ACLs disabled but domainId for DAG is specified" + ", aclsEnabled=true, domainId=" + domainId); } return null; } boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE, TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT); if (domainId != null) { // do nothing LOG.info("Using specified domainId with Timeline, domainId=" + domainId); return null; } else { if (!autoCreateDomain) { // Error - Cannot fallback to default as that leaves ACLs open throw new TezUncheckedException( "Timeline DomainId is not specified and auto-create" + " Domains is disabled"); } domainId = DOMAIN_ID_PREFIX + applicationId.toString() + "_" + dagName; createTimelineDomain(domainId, tezConf, dagAccessControls); LOG.info("Created Timeline Domain for DAG-specific History ACLs, domainId=" + domainId); return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, domainId); } }
From source file:org.apache.tez.dag.history.ats.acls.ATSV15HistoryACLPolicyManager.java
License:Apache License
private Map<String, String> createSessionDomain(Configuration tezConf, ApplicationId applicationId, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID); if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) { if (domainId != null) { throw new TezUncheckedException( "ACLs disabled but DomainId is specified" + ", aclsEnabled=true, domainId=" + domainId); }// w ww .ja v a2 s . c o m return null; } boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE, TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT); if (domainId != null) { // do nothing LOG.info("Using specified domainId with Timeline, domainId=" + domainId); return null; } else { if (!autoCreateDomain) { // Error - Cannot fallback to default as that leaves ACLs open throw new TezUncheckedException( "Timeline DomainId is not specified and auto-create" + " Domains is disabled"); } domainId = DOMAIN_ID_PREFIX + applicationId.toString(); createTimelineDomain(applicationId, domainId, tezConf, dagAccessControls); LOG.info("Created Timeline Domain for History ACLs, domainId=" + domainId); return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, domainId); } }
From source file:org.apache.tez.dag.history.ats.acls.ATSV15HistoryACLPolicyManager.java
License:Apache License
private Map<String, String> createDAGDomain(Configuration tezConf, ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { if (dagAccessControls == null) { // No DAG specific ACLs return null; }//from w w w . j a va 2 s . com String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID); if (!tezConf.getBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, TezConfiguration.TEZ_AM_ACLS_ENABLED_DEFAULT)) { if (domainId != null) { throw new TezUncheckedException("ACLs disabled but domainId for DAG is specified" + ", aclsEnabled=true, domainId=" + domainId); } return null; } boolean autoCreateDomain = tezConf.getBoolean(TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE, TezConfiguration.YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT); if (domainId != null) { // do nothing LOG.info("Using specified domainId with Timeline, domainId=" + domainId); return null; } else { if (!autoCreateDomain) { // Error - Cannot fallback to default as that leaves ACLs open throw new TezUncheckedException( "Timeline DomainId is not specified and auto-create" + " Domains is disabled"); } domainId = DOMAIN_ID_PREFIX + applicationId.toString() + "_" + dagName; createTimelineDomain(applicationId, domainId, tezConf, dagAccessControls); LOG.info("Created Timeline Domain for DAG-specific History ACLs, domainId=" + domainId); return Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, domainId); } }
From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryV15.java
License:Apache License
@Test public void testGetGroupId() { ApplicationId appId = ApplicationId.newInstance(1000l, 1); TezDAGID dagid = TezDAGID.getInstance(appId, 1); for (final HistoryEventType eventType : HistoryEventType.values()) { HistoryEvent historyEvent = new HistoryEvent() { @Override/*from w w w. ja v a 2 s . c o m*/ public HistoryEventType getEventType() { return eventType; } @Override public boolean isRecoveryEvent() { return false; } @Override public boolean isHistoryEvent() { return false; } @Override public void toProtoStream(OutputStream outputStream) throws IOException { } @Override public void fromProtoStream(InputStream inputStream) throws IOException { } }; DAGHistoryEvent event = new DAGHistoryEvent(dagid, historyEvent); ATSV15HistoryLoggingService service = new ATSV15HistoryLoggingService(); AppContext appContext = mock(AppContext.class); when(appContext.getApplicationID()).thenReturn(appId); service.setAppContext(appContext); TimelineEntityGroupId grpId = service.getGroupId(event); Assert.assertNotNull(grpId); Assert.assertEquals(appId, grpId.getApplicationId()); switch (eventType) { case AM_LAUNCHED: case APP_LAUNCHED: case AM_STARTED: case CONTAINER_LAUNCHED: case CONTAINER_STOPPED: Assert.assertEquals(appId.toString(), grpId.getTimelineEntityGroupId()); break; default: Assert.assertEquals(dagid.toString(), grpId.getTimelineEntityGroupId()); } } }
From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryWithACLs.java
License:Apache License
@Test(timeout = 50000) public void testSimpleAMACls() throws Exception { TezClient tezSession = null;/*from w w w.ja v a 2 s . co m*/ ApplicationId applicationId; String viewAcls = "nobody nobody_group"; try { SleepProcessorConfig spConf = new SleepProcessorConfig(1); DAG dag = DAG.create("TezSleepProcessor"); Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()) .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1)); dag.addVertex(vertex); TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls); tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); Path remoteStagingDir = remoteFs .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000)))); remoteFs.mkdirs(remoteStagingDir); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezSession = TezClient.create("TezSleepProcessor", tezConf, true); tezSession.start(); applicationId = tezSession.getAppMasterApplicationId(); DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState()); Thread.sleep(500l); dagStatus = dagClient.getDAGStatus(null); } assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState()); } finally { if (tezSession != null) { tezSession.stop(); } } TimelineDomain timelineDomain = getDomain( ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString()); verifyDomainACLs(timelineDomain, Collections.singleton("nobody"), Collections.singleton("nobody_group")); verifyEntityDomains(applicationId, true); }
From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryWithACLs.java
License:Apache License
@Test(timeout = 50000) public void testDAGACls() throws Exception { TezClient tezSession = null;/*from w w w . j av a2s. co m*/ ApplicationId applicationId; String viewAcls = "nobody nobody_group"; try { SleepProcessorConfig spConf = new SleepProcessorConfig(1); DAG dag = DAG.create("TezSleepProcessor"); Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(SleepProcessor.class.getName()) .setUserPayload(spConf.toUserPayload()), 1, Resource.newInstance(256, 1)); dag.addVertex(vertex); DAGAccessControls accessControls = new DAGAccessControls(); accessControls.setUsersWithViewACLs(Collections.singleton("nobody2")); accessControls.setGroupsWithViewACLs(Collections.singleton("nobody_group2")); dag.setAccessControls(accessControls); TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewAcls); tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); Path remoteStagingDir = remoteFs .makeQualified(new Path("/tmp", String.valueOf(random.nextInt(100000)))); remoteFs.mkdirs(remoteStagingDir); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezSession = TezClient.create("TezSleepProcessor", tezConf, true); tezSession.start(); applicationId = tezSession.getAppMasterApplicationId(); DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState()); Thread.sleep(500l); dagStatus = dagClient.getDAGStatus(null); } assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState()); } finally { if (tezSession != null) { tezSession.stop(); } } TimelineDomain timelineDomain = getDomain( ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString()); verifyDomainACLs(timelineDomain, Collections.singleton("nobody"), Collections.singleton("nobody_group")); timelineDomain = getDomain( ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString() + "_TezSleepProcessor"); verifyDomainACLs(timelineDomain, Sets.newHashSet("nobody", "nobody2"), Sets.newHashSet("nobody_group", "nobody_group2")); verifyEntityDomains(applicationId, false); }
From source file:org.apache.tez.dag.history.ats.acls.TestATSHistoryWithACLs.java
License:Apache License
private void verifyEntityDomains(ApplicationId applicationId, boolean sameDomain) { assertNotNull(timelineAddress);//from ww w . j a v a 2 s .c o m String appUrl = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_APPLICATION/" + "tez_" + applicationId.toString(); LOG.info("Getting timeline entity for tez application: " + appUrl); TimelineEntity appEntity = getTimelineData(appUrl, TimelineEntity.class); TezDAGID tezDAGID = TezDAGID.getInstance(applicationId, 1); String dagUrl = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + tezDAGID.toString(); LOG.info("Getting timeline entity for tez dag: " + dagUrl); TimelineEntity dagEntity = getTimelineData(dagUrl, TimelineEntity.class); // App and DAG entities should have different domains assertEquals(ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString(), appEntity.getDomainId()); if (!sameDomain) { assertEquals( ATSHistoryACLPolicyManager.DOMAIN_ID_PREFIX + applicationId.toString() + "_TezSleepProcessor", dagEntity.getDomainId()); } else { assertEquals(appEntity.getDomainId(), dagEntity.getDomainId()); } }