List of usage examples for org.apache.hadoop.security UserGroupInformation getCurrentUser
@InterfaceAudience.Public @InterfaceStability.Evolving public static UserGroupInformation getCurrentUser() throws IOException
From source file:org.apache.tez.dag.api.client.TestTimelineReaderFactory.java
License:Apache License
@Test(timeout = 5000) public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exception { ConnectionConfigurator connConf = mock(ConnectionConfigurator.class); TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory( connConf);/*from w w w. j ava 2s . c om*/ String inputUrl = "http://host:8080/path"; String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName(); HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl)); Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString()); }
From source file:org.apache.tez.dag.app.dag.impl.DAGImpl.java
License:Apache License
public DAGImpl(TezDAGID dagId, Configuration amConf, DAGPlan jobPlan, EventHandler eventHandler, TaskAttemptListener taskAttemptListener, Credentials dagCredentials, Clock clock, String appUserName, TaskHeartbeatHandler thh, AppContext appContext) { this.dagId = dagId; this.jobPlan = jobPlan; this.dagConf = new Configuration(amConf); Iterator<PlanKeyValuePair> iter = jobPlan.getDagConf().getConfKeyValuesList().iterator(); // override the amConf by using DAG level configuration while (iter.hasNext()) { PlanKeyValuePair keyValPair = iter.next(); TezConfiguration.validateProperty(keyValPair.getKey(), Scope.DAG); this.dagConf.set(keyValPair.getKey(), keyValPair.getValue()); }/* ww w . ja v a 2s. c om*/ this.dagName = (jobPlan.getName() != null) ? jobPlan.getName() : "<missing app name>"; this.userName = appUserName; this.clock = clock; this.appContext = appContext; this.taskAttemptListener = taskAttemptListener; this.taskHeartbeatHandler = thh; this.eventHandler = eventHandler; ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); this.localResources = DagTypeConverters.createLocalResourceMapFromDAGPlan(jobPlan.getLocalResourceList()); this.credentials = dagCredentials; if (this.credentials == null) { try { dagUGI = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new TezUncheckedException("Failed to set UGI for dag based on currentUser", e); } } else { dagUGI = UserGroupInformation.createRemoteUser(this.userName); dagUGI.addCredentials(this.credentials); } this.aclManager = new ACLManager(appContext.getAMACLManager(), dagUGI.getShortUserName(), this.dagConf); this.taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(dagConf); // This "this leak" is okay because the retained pointer is in an // instance variable. stateMachine = stateMachineFactory.make(this); this.entityUpdateTracker = new StateChangeNotifier(this); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" }) public void setupPostDagCreation() throws AMUserCodeException { String dagName = "dag0"; dispatcher = new DrainDispatcher(); appContext = mock(AppContext.class); thh = mock(TaskHeartbeatHandler.class); historyEventHandler = mock(HistoryEventHandler.class); TaskSchedulerEventHandler taskScheduler = mock(TaskSchedulerEventHandler.class); UserGroupInformation ugi;//from www . j a v a2 s .c o m try { ugi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new RuntimeException(e); } DAG dag = mock(DAG.class); doReturn(ugi).when(dag).getDagUGI(); doReturn(dagName).when(dag).getName(); doReturn(appAttemptId).when(appContext).getApplicationAttemptId(); doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID(); doReturn(dag).when(appContext).getCurrentDAG(); execService = mock(ListeningExecutorService.class); final ListenableFuture<Void> mockFuture = mock(ListenableFuture.class); Mockito.doAnswer(new Answer() { public ListenableFuture<Void> answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); CallableEvent e = (CallableEvent) args[0]; dispatcher.getEventHandler().handle(e); return mockFuture; } }).when(execService).submit((Callable<Void>) any()); doReturn(execService).when(appContext).getExecService(); doReturn(conf).when(appContext).getAMConf(); doReturn(new Credentials()).when(dag).getCredentials(); doReturn(DAGPlan.getDefaultInstance()).when(dag).getJobPlan(); doReturn(dagId).when(appContext).getCurrentDAGID(); doReturn(dagId).when(dag).getID(); doReturn(taskScheduler).when(appContext).getTaskScheduler(); doReturn(Resource.newInstance(102400, 60)).when(taskScheduler).getTotalResources(); doReturn(historyEventHandler).when(appContext).getHistoryHandler(); doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler(); vertexGroups = Maps.newHashMap(); for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) { vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo)); } updateTracker = new StateChangeNotifier(appContext.getCurrentDAG()); setupVertices(); when(dag.getVertex(any(TezVertexID.class))).thenAnswer(new Answer<Vertex>() { @Override public Vertex answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); if (args.length != 1) { return null; } TezVertexID vId = (TezVertexID) args[0]; return vertexIdMap.get(vId); } }); when(dag.getVertex(any(String.class))).thenAnswer(new Answer<Vertex>() { @Override public Vertex answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); if (args.length != 1) { return null; } String vId = (String) args[0]; return vertices.get(vId); } }); // TODO - this test logic is tightly linked to impl DAGImpl code. edges = new HashMap<String, Edge>(); for (EdgePlan edgePlan : dagPlan.getEdgeList()) { EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(edgePlan); edges.put(edgePlan.getId(), new Edge(edgeProperty, dispatcher.getEventHandler())); } parseVertexEdges(); for (Edge edge : edges.values()) { edge.initialize(); } dispatcher.register(CallableEventType.class, new CallableEventDispatcher()); taskAttemptEventDispatcher = new TaskAttemptEventDispatcher(); dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher); taskEventDispatcher = new TaskEventDispatcher(); dispatcher.register(TaskEventType.class, taskEventDispatcher); vertexEventDispatcher = new VertexEventDispatcher(); dispatcher.register(VertexEventType.class, vertexEventDispatcher); dagEventDispatcher = new DagEventDispatcher(); dispatcher.register(DAGEventType.class, dagEventDispatcher); dispatcher.init(conf); dispatcher.start(); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
@Test(timeout = 5000) public void testVertexWithOneToOneSplitWhileRunning() throws Exception { int numTasks = 5; // create a diamond shaped dag with 1-1 edges. setupPreDagCreation();//from w w w .j a v a2 s . c om dagPlan = createDAGPlanForOneToOneSplit(null, numTasks, false); setupPostDagCreation(); VertexImpl v1 = vertices.get("vertex1"); v1.vertexReconfigurationPlanned(); initAllVertices(VertexState.INITED); // fudge vertex manager so that tasks dont start running v1.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), v1, appContext, mock(StateChangeNotifier.class)); v1.vertexManager.initialize(); startVertex(v1); dispatcher.await(); Assert.assertEquals(numTasks, vertices.get("vertex2").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex3").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex4").getTotalTasks()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex1").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex2").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); // change parallelism int newNumTasks = 3; v1.setParallelism(newNumTasks, null, null, null, true); v1.doneReconfiguringVertex(); dispatcher.await(); Assert.assertEquals(newNumTasks, vertices.get("vertex2").getTotalTasks()); Assert.assertEquals(newNumTasks, vertices.get("vertex3").getTotalTasks()); Assert.assertEquals(newNumTasks, vertices.get("vertex4").getTotalTasks()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex1").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex2").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
@Test(timeout = 5000) public void testVertexWithOneToOneSplitWhileInited() throws Exception { int numTasks = 5; // create a diamond shaped dag with 1-1 edges. setupPreDagCreation();/*from www . j a v a 2s . c o m*/ dagPlan = createDAGPlanForOneToOneSplit(null, numTasks, false); setupPostDagCreation(); VertexImpl v1 = vertices.get("vertex1"); v1.vertexReconfigurationPlanned(); initAllVertices(VertexState.INITED); // fudge vertex manager so that tasks dont start running v1.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), v1, appContext, mock(StateChangeNotifier.class)); v1.vertexManager.initialize(); Assert.assertEquals(numTasks, vertices.get("vertex2").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex3").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex4").getTotalTasks()); // change parallelism int newNumTasks = 3; v1.setParallelism(newNumTasks, null, null, null, true); v1.doneReconfiguringVertex(); dispatcher.await(); Assert.assertEquals(newNumTasks, vertices.get("vertex2").getTotalTasks()); Assert.assertEquals(newNumTasks, vertices.get("vertex3").getTotalTasks()); Assert.assertEquals(newNumTasks, vertices.get("vertex4").getTotalTasks()); Assert.assertEquals(VertexState.INITED, vertices.get("vertex1").getState()); Assert.assertEquals(VertexState.INITED, vertices.get("vertex2").getState()); Assert.assertEquals(VertexState.INITED, vertices.get("vertex3").getState()); Assert.assertEquals(VertexState.INITED, vertices.get("vertex4").getState()); startVertex(v1); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex1").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex2").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from w w w .j a v a 2s .c o m*/ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { // Race when a source vertex manages to start before the target vertex has // been initialized setupPreDagCreation(); dagPlan = createSamplerDAGPlan(true); setupPostDagCreation(); VertexImpl vA = vertices.get("A"); VertexImpl vB = vertices.get("B"); VertexImpl vC = vertices.get("C"); TestUpdateListener listener = new TestUpdateListener(); updateTracker.registerForVertexUpdates(vB.getName(), EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED), listener); // fudge the vm so we can do custom stuff vB.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), vB, appContext, mock(StateChangeNotifier.class)); vB.vertexReconfigurationPlanned(); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_INIT)); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_START)); dispatcher.await(); Assert.assertEquals(VertexState.INITIALIZING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); // setting the edge manager should vA to start EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor .create(EdgeManagerForTest.class.getName()); Edge e = vC.sourceVertices.get(vA); Assert.assertNull(e.getEdgeManager()); e.setCustomEdgeManager(mockEdgeManagerDescriptor); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); // vB is not configured yet. Edge to C is not configured. So it should not send configured event // even thought VM says its doneConfiguring vertex vB.doneReconfiguringVertex(); Assert.assertEquals(0, listener.events.size()); // complete configuration and verify getting configured signal from vB Map<String, EdgeManagerPluginDescriptor> edges = Maps.newHashMap(); edges.put("B", mockEdgeManagerDescriptor); vC.setParallelism(2, vertexLocationHint, edges, null, true); dispatcher.await(); Assert.assertEquals(1, listener.events.size()); Assert.assertEquals(vB.getName(), listener.events.get(0).getVertexName()); Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, listener.events.get(0).getVertexState()); updateTracker.unregisterForVertexUpdates(vB.getName(), listener); Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.RUNNING, vB.getState()); Assert.assertEquals(VertexState.RUNNING, vC.getState()); Assert.assertNotNull(vA.getTask(0)); Assert.assertNotNull(vB.getTask(0)); Assert.assertNotNull(vC.getTask(0)); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexManager.java
License:Apache License
@Test(timeout = 5000) public void testOnRootVertexInitialized() throws Exception { VertexManager vm = new VertexManager( VertexManagerPluginDescriptor.create(RootInputVertexManager.class.getName()), UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, mock(StateChangeNotifier.class)); vm.initialize();/*from w ww .j a v a 2 s. co m*/ InputDescriptor id1 = mock(InputDescriptor.class); List<Event> events1 = new LinkedList<Event>(); InputDataInformationEvent diEvent1 = InputDataInformationEvent.createWithSerializedPayload(0, null); events1.add(diEvent1); vm.onRootVertexInitialized("input1", id1, events1); verify(mockHandler, times(1)).handle(requestCaptor.capture()); List<TezEvent> tezEvents1 = requestCaptor.getValue().getEvents(); assertEquals(1, tezEvents1.size()); assertEquals(diEvent1, tezEvents1.get(0).getEvent()); InputDescriptor id2 = mock(InputDescriptor.class); List<Event> events2 = new LinkedList<Event>(); InputDataInformationEvent diEvent2 = InputDataInformationEvent.createWithSerializedPayload(0, null); events2.add(diEvent2); vm.onRootVertexInitialized("input1", id2, events2); verify(mockHandler, times(2)).handle(requestCaptor.capture()); List<TezEvent> tezEvents2 = requestCaptor.getValue().getEvents(); assertEquals(tezEvents2.size(), 1); assertEquals(diEvent2, tezEvents2.get(0).getEvent()); }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexManager.java
License:Apache License
/** * TEZ-1647//w ww.j a va 2 s.c o m * custom vertex manager generates events only when both i1 and i2 are initialized. * @throws Exception */ @Test(timeout = 5000) public void testOnRootVertexInitialized2() throws Exception { VertexManager vm = new VertexManager( VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, mock(StateChangeNotifier.class)); vm.initialize(); InputDescriptor id1 = mock(InputDescriptor.class); List<Event> events1 = new LinkedList<Event>(); InputDataInformationEvent diEvent1 = InputDataInformationEvent.createWithSerializedPayload(0, null); events1.add(diEvent1); // do not call context.addRootInputEvents, just cache the TezEvent vm.onRootVertexInitialized("input1", id1, events1); verify(mockHandler, times(1)).handle(requestCaptor.capture()); List<TezEvent> tezEventsAfterInput1 = requestCaptor.getValue().getEvents(); assertEquals(0, tezEventsAfterInput1.size()); InputDescriptor id2 = mock(InputDescriptor.class); List<Event> events2 = new LinkedList<Event>(); InputDataInformationEvent diEvent2 = InputDataInformationEvent.createWithSerializedPayload(0, null); events2.add(diEvent2); // call context.addRootInputEvents(input1), context.addRootInputEvents(input2) vm.onRootVertexInitialized("input2", id2, events2); verify(mockHandler, times(2)).handle(requestCaptor.capture()); List<TezEvent> tezEventsAfterInput2 = requestCaptor.getValue().getEvents(); assertEquals(2, tezEventsAfterInput2.size()); // also verify the EventMetaData Set<String> edgeVertexSet = new HashSet<String>(); for (TezEvent tezEvent : tezEventsAfterInput2) { edgeVertexSet.add(tezEvent.getDestinationInfo().getEdgeVertexName()); } assertEquals(Sets.newHashSet("input1", "input2"), edgeVertexSet); }
From source file:org.apache.tez.dag.app.DAGAppMaster.java
License:Apache License
public static void main(String[] args) { try {//from ww w . j ava2 s. c o m Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); String containerIdStr = System.getenv(Environment.CONTAINER_ID.name()); String nodeHostString = System.getenv(Environment.NM_HOST.name()); String nodePortString = System.getenv(Environment.NM_PORT.name()); String nodeHttpPortString = System.getenv(Environment.NM_HTTP_PORT.name()); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); String clientVersion = System.getenv(TezConstants.TEZ_CLIENT_VERSION_ENV); if (clientVersion == null) { clientVersion = VersionInfo.UNKNOWN; } // TODO Should this be defaulting to 1. Was there a version of YARN where this was not setup ? int maxAppAttempts = 1; String maxAppAttemptsEnv = System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV); if (maxAppAttemptsEnv != null) { maxAppAttempts = Integer.valueOf(maxAppAttemptsEnv); } validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); long appSubmitTime = Long.parseLong(appSubmitTimeStr); String jobUserName = System.getenv(ApplicationConstants.Environment.USER.name()); // Command line options Options opts = new Options(); opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION, false, "Run Tez Application Master in Session mode"); CommandLine cliParser = new GnuParser().parse(opts, args); // TODO Does this really need to be a YarnConfiguration ? Configuration conf = new Configuration(new YarnConfiguration()); TezUtilsInternal.addUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()), conf); UserGroupInformation.setConfiguration(conf); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DAGAppMaster appMaster = new DAGAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), Integer.parseInt(nodeHttpPortString), new SystemClock(), appSubmitTime, cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION), System.getenv(Environment.PWD.name()), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())), TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())), clientVersion, maxAppAttempts, credentials, jobUserName); ShutdownHookManager.get().addShutdownHook(new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); initAndStartAppMaster(appMaster, conf); } catch (Throwable t) { LOG.fatal("Error starting DAGAppMaster", t); System.exit(1); } }
From source file:org.apache.tez.dag.app.launcher.TezLocalCacheManager.java
License:Apache License
public TezLocalCacheManager(Map<String, LocalResource> resources, Configuration conf) throws IOException { this.ugi = UserGroupInformation.getCurrentUser(); this.fileContext = FileContext.getLocalFSFileContext(); this.resources = resources; this.conf = conf; this.tempDir = Files.createTempDirectory(Paths.get("."), "tez-local-cache"); }