List of usage examples for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED
FinalApplicationStatus SUCCEEDED
To view the source code for org.apache.hadoop.yarn.api.records FinalApplicationStatus SUCCEEDED.
Click Source Link
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 50000) public void testPreemptionNoHeadroom() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE, 10); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 3); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, 60 * 1000); // vertex 0 and vertex 2 are root vertices and vertex 1 is a child of vertex 0 DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(3); when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[] { 0x2 })); when(mockDagInfo.getVertexDescendants(1)).thenReturn(new BitSet()); when(mockDagInfo.getVertexDescendants(2)).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();/* www .ja v a 2 s . co m*/ drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); Priority priorityv2 = Priority.newInstance(3); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t0.task)).thenReturn(0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t1.task)).thenReturn(0); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv1t0.task)).thenReturn(1); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv1t1.task)).thenReturn(1); MockTaskInfo taskv2t0 = new MockTaskInfo("taskv2t0", priorityv2, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv2t0.task)).thenReturn(2); // asks for two tasks for vertex 1 and start running one of them TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); TaskRequest reqv1t1 = taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv1t0.capability, priorityv1, null); scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // start running the other task for vertex 1 a bit later clock.incrementTime(1000); ContainerId cid2 = ContainerId.newContainerId(attemptId, 2); Container container2 = Container.newInstance(cid2, host1, null, taskv1t0.capability, priorityv1, null); scheduler.onContainersAllocated(Collections.singletonList(container2)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv1t1.task, taskv1t1.cookie, container2); verify(mockRMClient).removeContainerRequest(reqv1t1); // add a request for vertex 0 but there is no headroom when(mockRMClient.getAvailableResources()).thenReturn(Resources.none()); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); // should preempt after enough heartbeats to get past preemption interval // only the youngest container should be preempted to meet the demand scheduler.getProgress(); scheduler.getProgress(); scheduler.getProgress(); drainableAppCallback.drain(); verify(mockApp, times(1)).preemptContainer(any(ContainerId.class)); verify(mockApp).preemptContainer(cid2); assertEquals(taskv1t1.task, scheduler.deallocateContainer(cid2)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid2); verify(mockRMClient).releaseAssignedContainer(cid2); verify(mockApp, never()).containerBeingReleased(cid1); verify(mockRMClient, never()).releaseAssignedContainer(cid1); // add a request for vertex 2 and allocate another container clock.incrementTime(1000); taskRequestCaptor.scheduleTask(taskv2t0); ContainerId cid3 = ContainerId.newContainerId(attemptId, 3); Container container3 = Container.newInstance(cid3, host1, null, taskv0t0.capability, priorityv0, null); scheduler.onContainersAllocated(Collections.singletonList(container3)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container3); verify(mockRMClient).removeContainerRequest(reqv0t0); // no more preemptions since v1 is not a descendant of v2 scheduler.getProgress(); scheduler.getProgress(); scheduler.getProgress(); drainableAppCallback.drain(); verify(mockApp, times(1)).preemptContainer(any(ContainerId.class)); // adding request for v0 should trigger preemption on next heartbeat taskRequestCaptor.scheduleTask(taskv0t1); scheduler.getProgress(); drainableAppCallback.drain(); verify(mockApp, times(2)).preemptContainer(any(ContainerId.class)); verify(mockApp).preemptContainer(cid1); assertEquals(taskv1t0.task, scheduler.deallocateContainer(cid1)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked" }) @Test(timeout = 10000)/* ww w . ja v a2 s. co m*/ public void testTaskSchedulerNoReuse() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); TaskSchedulerAppCallbackDrainable drainableAppCallback = scheduler.getDrainableAppCallback(); Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); int interval = 100; conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, interval); scheduler.init(conf); drainableAppCallback.drain(); verify(mockRMClient).init(conf); verify(mockRMClient).setHeartbeatInterval(interval); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); ByteBuffer mockKey = mock(ByteBuffer.class); when(mockRegResponse.getClientToAMTokenMasterKey()).thenReturn(mockKey); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); verify(mockApp).setApplicationRegistrationData(mockMaxResource, mockAcls, mockKey); when(mockRMClient.getClusterNodeCount()).thenReturn(5); Assert.assertEquals(5, scheduler.getClusterNodeCount()); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); Assert.assertEquals(mockClusterResource, mockRMClient.getAvailableResources()); Object mockTask1 = mock(Object.class); Object mockCookie1 = mock(Object.class); Resource mockCapability = mock(Resource.class); String[] hosts = { "host1", "host5" }; String[] racks = { "/default-rack", "/default-rack" }; Priority mockPriority = mock(Priority.class); ArgumentCaptor<CookieContainerRequest> requestCaptor = ArgumentCaptor .forClass(CookieContainerRequest.class); // allocate task scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(1)).addContainerRequest((CookieContainerRequest) any()); // returned from task requests before allocation happens assertFalse(scheduler.deallocateTask(mockTask1, true)); verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, times(1)).removeContainerRequest((CookieContainerRequest) any()); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // deallocating unknown task assertFalse(scheduler.deallocateTask(mockTask1, true)); verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, times(1)).removeContainerRequest((CookieContainerRequest) any()); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // allocate tasks Object mockTask2 = mock(Object.class); Object mockCookie2 = mock(Object.class); Object mockTask3 = mock(Object.class); Object mockCookie3 = mock(Object.class); scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request1 = requestCaptor.getValue(); scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority, null, mockCookie3); drainableAppCallback.drain(); verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request3 = requestCaptor.getValue(); List<Container> containers = new ArrayList<Container>(); Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer1.getNodeId().getHost()).thenReturn("host1"); ContainerId mockCId1 = mock(ContainerId.class); when(mockContainer1.getId()).thenReturn(mockCId1); containers.add(mockContainer1); Container mockContainer2 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer2.getNodeId().getHost()).thenReturn("host2"); ContainerId mockCId2 = mock(ContainerId.class); when(mockContainer2.getId()).thenReturn(mockCId2); containers.add(mockContainer2); Container mockContainer3 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer3.getNodeId().getHost()).thenReturn("host3"); ContainerId mockCId3 = mock(ContainerId.class); when(mockContainer3.getId()).thenReturn(mockCId3); containers.add(mockContainer3); Container mockContainer4 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer4.getNodeId().getHost()).thenReturn("host4"); ContainerId mockCId4 = mock(ContainerId.class); when(mockContainer4.getId()).thenReturn(mockCId4); containers.add(mockContainer4); ArrayList<CookieContainerRequest> hostContainers = new ArrayList<CookieContainerRequest>(); hostContainers.add(request1); hostContainers.add(request2); hostContainers.add(request3); ArrayList<CookieContainerRequest> rackContainers = new ArrayList<CookieContainerRequest>(); rackContainers.add(request2); rackContainers.add(request3); ArrayList<CookieContainerRequest> anyContainers = new ArrayList<CookieContainerRequest>(); anyContainers.add(request3); final List<ArrayList<CookieContainerRequest>> hostList = new LinkedList<ArrayList<CookieContainerRequest>>(); hostList.add(hostContainers); final List<ArrayList<CookieContainerRequest>> rackList = new LinkedList<ArrayList<CookieContainerRequest>>(); rackList.add(rackContainers); final List<ArrayList<CookieContainerRequest>> anyList = new LinkedList<ArrayList<CookieContainerRequest>>(); anyList.add(anyContainers); final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>(); // return all requests for host1 when(mockRMClient.getMatchingRequests((Priority) any(), eq("host1"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return hostList; } }); // first request matched by host // second request matched to rack. RackResolver by default puts hosts in // /default-rack. We need to workaround by returning rack matches only once when(mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return rackList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); // third request matched to ANY when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return anyList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); // first container allocated verify(mockApp).taskAllocated(mockTask1, mockCookie1, mockContainer1); verify(mockApp).taskAllocated(mockTask2, mockCookie2, mockContainer2); verify(mockApp).taskAllocated(mockTask3, mockCookie3, mockContainer3); // no other allocations returned verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any()); verify(mockRMClient).removeContainerRequest(request1); verify(mockRMClient).removeContainerRequest(request2); verify(mockRMClient).removeContainerRequest(request3); // verify unwanted container released verify(mockRMClient).releaseAssignedContainer(mockCId4); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask1, true)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId1); verify(mockRMClient).releaseAssignedContainer(mockCId1); // deallocate allocated container Assert.assertEquals(mockTask2, scheduler.deallocateContainer(mockCId2)); drainableAppCallback.drain(); verify(mockRMClient).releaseAssignedContainer(mockCId2); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); List<ContainerStatus> statuses = new ArrayList<ContainerStatus>(); ContainerStatus mockStatus1 = mock(ContainerStatus.class); when(mockStatus1.getContainerId()).thenReturn(mockCId1); statuses.add(mockStatus1); ContainerStatus mockStatus2 = mock(ContainerStatus.class); when(mockStatus2.getContainerId()).thenReturn(mockCId2); statuses.add(mockStatus2); ContainerStatus mockStatus3 = mock(ContainerStatus.class); when(mockStatus3.getContainerId()).thenReturn(mockCId3); statuses.add(mockStatus3); ContainerStatus mockStatus4 = mock(ContainerStatus.class); when(mockStatus4.getContainerId()).thenReturn(mockCId4); statuses.add(mockStatus4); scheduler.onContainersCompleted(statuses); drainableAppCallback.drain(); // released container status returned verify(mockApp).containerCompleted(mockTask1, mockStatus1); verify(mockApp).containerCompleted(mockTask2, mockStatus2); // currently allocated container status returned and not released verify(mockApp).containerCompleted(mockTask3, mockStatus3); // no other statuses returned verify(mockApp, times(3)).containerCompleted(any(), (ContainerStatus) any()); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); // verify blacklisting verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId) any()); String badHost = "host6"; NodeId badNodeId = mock(NodeId.class); when(badNodeId.getHost()).thenReturn(badHost); scheduler.blacklistNode(badNodeId); verify(mockRMClient, times(1)).addNodeToBlacklist(badNodeId); Object mockTask4 = mock(Object.class); Object mockCookie4 = mock(Object.class); scheduler.allocateTask(mockTask4, mockCapability, null, null, mockPriority, null, mockCookie4); drainableAppCallback.drain(); verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request4 = requestCaptor.getValue(); anyContainers.clear(); anyContainers.add(request4); Container mockContainer5 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer5.getNodeId().getHost()).thenReturn(badHost); when(mockContainer5.getNodeId()).thenReturn(badNodeId); ContainerId mockCId5 = mock(ContainerId.class); when(mockContainer5.getId()).thenReturn(mockCId5); containers.clear(); containers.add(mockContainer5); when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return anyList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); // no new allocation verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any()); // verify blacklisted container released verify(mockRMClient).releaseAssignedContainer(mockCId5); verify(mockRMClient, times(4)).releaseAssignedContainer((ContainerId) any()); // verify request added back verify(mockRMClient, times(6)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request5 = requestCaptor.getValue(); anyContainers.clear(); anyContainers.add(request5); Container mockContainer6 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer6.getNodeId().getHost()).thenReturn("host7"); ContainerId mockCId6 = mock(ContainerId.class); when(mockContainer6.getId()).thenReturn(mockCId6); containers.clear(); containers.add(mockContainer6); when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return anyList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); // new allocation verify(mockApp, times(4)).taskAllocated(any(), any(), (Container) any()); verify(mockApp).taskAllocated(mockTask4, mockCookie4, mockContainer6); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask4, true)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId6); verify(mockRMClient).releaseAssignedContainer(mockCId6); verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); // test unblacklist scheduler.unblacklistNode(badNodeId); verify(mockRMClient, times(1)).removeNodeFromBlacklist(badNodeId); assertEquals(0, scheduler.blacklistedNodes.size()); float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); Assert.assertEquals(progress, scheduler.getProgress(), 0); // check duplicate allocation request scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(7)).addContainerRequest((CookieContainerRequest) any()); verify(mockRMClient, times(6)).removeContainerRequest((CookieContainerRequest) any()); scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); // old request removed and new one added verify(mockRMClient, times(7)).removeContainerRequest((CookieContainerRequest) any()); verify(mockRMClient, times(8)).addContainerRequest((CookieContainerRequest) any()); assertFalse(scheduler.deallocateTask(mockTask1, true)); List<NodeReport> mockUpdatedNodes = mock(List.class); scheduler.onNodesUpdated(mockUpdatedNodes); drainableAppCallback.drain(); verify(mockApp).nodesUpdated(mockUpdatedNodes); Exception mockException = mock(Exception.class); scheduler.onError(mockException); drainableAppCallback.drain(); verify(mockApp).onError(mockException); scheduler.onShutdownRequest(); drainableAppCallback.drain(); verify(mockApp).appShutdownRequested(); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.stop(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); scheduler.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked" }) @Test(timeout = 10000)//from w w w. ja v a 2 s . c o m public void testTaskSchedulerWithReuse() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); final TaskSchedulerAppCallbackDrainable drainableAppCallback = scheduler.getDrainableAppCallback(); Configuration conf = new Configuration(); // to match all in the same pass conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 0); // to release immediately after deallocate conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 0); scheduler.init(conf); drainableAppCallback.drain(); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); scheduler.start(); drainableAppCallback.drain(); Object mockTask1 = mock(Object.class); when(mockTask1.toString()).thenReturn("task1"); Object mockCookie1 = mock(Object.class); Resource mockCapability = mock(Resource.class); String[] hosts = { "host1", "host5" }; String[] racks = { "/default-rack", "/default-rack" }; final Priority mockPriority1 = Priority.newInstance(1); final Priority mockPriority2 = Priority.newInstance(2); final Priority mockPriority3 = Priority.newInstance(3); final Priority mockPriority4 = Priority.newInstance(4); final Priority mockPriority5 = Priority.newInstance(5); Object mockTask2 = mock(Object.class); when(mockTask2.toString()).thenReturn("task2"); Object mockCookie2 = mock(Object.class); Object mockTask3 = mock(Object.class); when(mockTask3.toString()).thenReturn("task3"); Object mockCookie3 = mock(Object.class); ArgumentCaptor<CookieContainerRequest> requestCaptor = ArgumentCaptor .forClass(CookieContainerRequest.class); scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority1, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(1)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request1 = requestCaptor.getValue(); scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority2, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority3, null, mockCookie3); drainableAppCallback.drain(); verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request3 = requestCaptor.getValue(); List<Container> containers = new ArrayList<Container>(); // sending lower priority container first to make sure its not matched Container mockContainer4 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer4.getNodeId().getHost()).thenReturn("host4"); when(mockContainer4.toString()).thenReturn("container4"); when(mockContainer4.getPriority()).thenReturn(mockPriority4); ContainerId mockCId4 = mock(ContainerId.class); when(mockContainer4.getId()).thenReturn(mockCId4); when(mockCId4.toString()).thenReturn("container4"); containers.add(mockContainer4); Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer1.getNodeId().getHost()).thenReturn("host1"); when(mockContainer1.getPriority()).thenReturn(mockPriority1); when(mockContainer1.toString()).thenReturn("container1"); ContainerId mockCId1 = mock(ContainerId.class); when(mockContainer1.getId()).thenReturn(mockCId1); when(mockCId1.toString()).thenReturn("container1"); containers.add(mockContainer1); Container mockContainer2 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer2.getNodeId().getHost()).thenReturn("host2"); when(mockContainer2.getPriority()).thenReturn(mockPriority2); when(mockContainer2.toString()).thenReturn("container2"); ContainerId mockCId2 = mock(ContainerId.class); when(mockContainer2.getId()).thenReturn(mockCId2); when(mockCId2.toString()).thenReturn("container2"); containers.add(mockContainer2); Container mockContainer3 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer3.getNodeId().getHost()).thenReturn("host3"); when(mockContainer3.getPriority()).thenReturn(mockPriority3); when(mockContainer3.toString()).thenReturn("container3"); ContainerId mockCId3 = mock(ContainerId.class); when(mockContainer3.getId()).thenReturn(mockCId3); when(mockCId3.toString()).thenReturn("container3"); containers.add(mockContainer3); ArrayList<CookieContainerRequest> hostContainers = new ArrayList<CookieContainerRequest>(); hostContainers.add(request1); ArrayList<CookieContainerRequest> rackContainers = new ArrayList<CookieContainerRequest>(); rackContainers.add(request2); ArrayList<CookieContainerRequest> anyContainers = new ArrayList<CookieContainerRequest>(); anyContainers.add(request3); final List<ArrayList<CookieContainerRequest>> hostList = new LinkedList<ArrayList<CookieContainerRequest>>(); hostList.add(hostContainers); final List<ArrayList<CookieContainerRequest>> rackList = new LinkedList<ArrayList<CookieContainerRequest>>(); rackList.add(rackContainers); final List<ArrayList<CookieContainerRequest>> anyList = new LinkedList<ArrayList<CookieContainerRequest>>(); anyList.add(anyContainers); final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>(); // return pri1 requests for host1 when(mockRMClient.getMatchingRequestsForTopPriority(eq("host1"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return hostList; } }); // second request matched to rack. RackResolver by default puts hosts in // /default-rack. We need to workaround by returning rack matches only once when(mockRMClient.getMatchingRequestsForTopPriority(eq("/default-rack"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return rackList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); // third request matched to ANY when(mockRMClient.getMatchingRequestsForTopPriority(eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return anyList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); when(mockRMClient.getTopPriority()).then(new Answer<Priority>() { @Override public Priority answer(InvocationOnMock invocation) throws Throwable { int allocations = drainableAppCallback.count.get(); if (allocations == 0) { return mockPriority1; } if (allocations == 1) { return mockPriority2; } if (allocations == 2) { return mockPriority3; } if (allocations == 3) { return mockPriority4; } return null; } }); AtomicBoolean drainNotifier = new AtomicBoolean(false); scheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; scheduler.onContainersAllocated(containers); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); // exact number allocations returned verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any()); // first container allocated verify(mockApp).taskAllocated(mockTask1, mockCookie1, mockContainer1); verify(mockApp).taskAllocated(mockTask2, mockCookie2, mockContainer2); verify(mockApp).taskAllocated(mockTask3, mockCookie3, mockContainer3); verify(mockRMClient).removeContainerRequest(request1); verify(mockRMClient).removeContainerRequest(request2); verify(mockRMClient).removeContainerRequest(request3); // verify unwanted container released verify(mockRMClient).releaseAssignedContainer(mockCId4); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask1, true)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId1); verify(mockRMClient).releaseAssignedContainer(mockCId1); // deallocate allocated container Assert.assertEquals(mockTask2, scheduler.deallocateContainer(mockCId2)); drainableAppCallback.drain(); verify(mockRMClient).releaseAssignedContainer(mockCId2); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); List<ContainerStatus> statuses = new ArrayList<ContainerStatus>(); ContainerStatus mockStatus1 = mock(ContainerStatus.class); when(mockStatus1.getContainerId()).thenReturn(mockCId1); statuses.add(mockStatus1); ContainerStatus mockStatus2 = mock(ContainerStatus.class); when(mockStatus2.getContainerId()).thenReturn(mockCId2); statuses.add(mockStatus2); ContainerStatus mockStatus3 = mock(ContainerStatus.class); when(mockStatus3.getContainerId()).thenReturn(mockCId3); statuses.add(mockStatus3); ContainerStatus mockStatus4 = mock(ContainerStatus.class); when(mockStatus4.getContainerId()).thenReturn(mockCId4); statuses.add(mockStatus4); scheduler.onContainersCompleted(statuses); drainableAppCallback.drain(); // released container status returned verify(mockApp).containerCompleted(mockTask1, mockStatus1); verify(mockApp).containerCompleted(mockTask2, mockStatus2); // currently allocated container status returned and not released verify(mockApp).containerCompleted(mockTask3, mockStatus3); // no other statuses returned verify(mockApp, times(3)).containerCompleted(any(), (ContainerStatus) any()); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); // verify blacklisting verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId) any()); String badHost = "host6"; NodeId badNodeId = mock(NodeId.class); when(badNodeId.getHost()).thenReturn(badHost); scheduler.blacklistNode(badNodeId); verify(mockRMClient, times(1)).addNodeToBlacklist(badNodeId); Object mockTask4 = mock(Object.class); when(mockTask4.toString()).thenReturn("task4"); Object mockCookie4 = mock(Object.class); scheduler.allocateTask(mockTask4, mockCapability, null, null, mockPriority4, null, mockCookie4); drainableAppCallback.drain(); verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request4 = requestCaptor.getValue(); anyContainers.clear(); anyContainers.add(request4); Container mockContainer5 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer5.getNodeId().getHost()).thenReturn(badHost); when(mockContainer5.getNodeId()).thenReturn(badNodeId); ContainerId mockCId5 = mock(ContainerId.class); when(mockContainer5.toString()).thenReturn("container5"); when(mockCId5.toString()).thenReturn("container5"); when(mockContainer5.getId()).thenReturn(mockCId5); when(mockContainer5.getPriority()).thenReturn(mockPriority4); containers.clear(); containers.add(mockContainer5); when(mockRMClient.getMatchingRequestsForTopPriority(eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return anyList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); drainNotifier.set(false); scheduler.onContainersAllocated(containers); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); // no new allocation verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any()); // verify blacklisted container released verify(mockRMClient).releaseAssignedContainer(mockCId5); verify(mockRMClient, times(4)).releaseAssignedContainer((ContainerId) any()); // verify request added back verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request5 = requestCaptor.getValue(); anyContainers.clear(); anyContainers.add(request5); Container mockContainer6 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer6.getNodeId().getHost()).thenReturn("host7"); ContainerId mockCId6 = mock(ContainerId.class); when(mockContainer6.getId()).thenReturn(mockCId6); when(mockContainer6.toString()).thenReturn("container6"); when(mockCId6.toString()).thenReturn("container6"); containers.clear(); containers.add(mockContainer6); when(mockRMClient.getMatchingRequestsForTopPriority(eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return anyList; } }).thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); drainNotifier.set(false); scheduler.onContainersAllocated(containers); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); // new allocation verify(mockApp, times(4)).taskAllocated(any(), any(), (Container) any()); verify(mockApp).taskAllocated(mockTask4, mockCookie4, mockContainer6); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask4, true)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId6); verify(mockRMClient).releaseAssignedContainer(mockCId6); verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); // test unblacklist scheduler.unblacklistNode(badNodeId); verify(mockRMClient, times(1)).removeNodeFromBlacklist(badNodeId); assertEquals(0, scheduler.blacklistedNodes.size()); // verify container level matching // fudge the top level priority to prevent containers from being released // if top level priority is higher than newly allocated containers then // they will not be released final AtomicBoolean fudgePriority = new AtomicBoolean(true); when(mockRMClient.getTopPriority()).then(new Answer<Priority>() { @Override public Priority answer(InvocationOnMock invocation) throws Throwable { if (fudgePriority.get()) { return mockPriority4; } return mockPriority5; } }); // add a dummy task to prevent release of allocated containers Object mockTask5 = mock(Object.class); when(mockTask5.toString()).thenReturn("task5"); Object mockCookie5 = mock(Object.class); scheduler.allocateTask(mockTask5, mockCapability, hosts, racks, mockPriority5, null, mockCookie5); verify(mockRMClient, times(6)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request6 = requestCaptor.getValue(); drainableAppCallback.drain(); // add containers so that we can reference one of them for container specific // allocation containers.clear(); Container mockContainer7 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer7.getNodeId().getHost()).thenReturn("host5"); ContainerId mockCId7 = mock(ContainerId.class); when(mockContainer7.toString()).thenReturn("container7"); when(mockCId7.toString()).thenReturn("container7"); when(mockContainer7.getId()).thenReturn(mockCId7); when(mockContainer7.getPriority()).thenReturn(mockPriority5); containers.add(mockContainer7); Container mockContainer8 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer8.getNodeId().getHost()).thenReturn("host5"); ContainerId mockCId8 = mock(ContainerId.class); when(mockContainer8.toString()).thenReturn("container8"); when(mockCId8.toString()).thenReturn("container8"); when(mockContainer8.getId()).thenReturn(mockCId8); when(mockContainer8.getPriority()).thenReturn(mockPriority5); containers.add(mockContainer8); drainNotifier.set(false); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); Object mockTask6 = mock(Object.class); when(mockTask6.toString()).thenReturn("task6"); Object mockCookie6 = mock(Object.class); // allocate request with container affinity scheduler.allocateTask(mockTask6, mockCapability, mockCId7, mockPriority5, null, mockCookie6); drainableAppCallback.drain(); verify(mockRMClient, times(7)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest request7 = requestCaptor.getValue(); hostContainers.clear(); hostContainers.add(request6); hostContainers.add(request7); when(mockRMClient.getMatchingRequestsForTopPriority(eq("host5"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return hostList; } }); // stop fudging top priority fudgePriority.set(false); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); verify(mockApp, times(6)).taskAllocated(any(), any(), (Container) any()); // container7 allocated to the task with affinity for it verify(mockApp).taskAllocated(mockTask6, mockCookie6, mockContainer7); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask5, true)); assertTrue(scheduler.deallocateTask(mockTask6, true)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId7); verify(mockApp).containerBeingReleased(mockCId8); verify(mockRMClient).releaseAssignedContainer(mockCId7); verify(mockRMClient).releaseAssignedContainer(mockCId8); verify(mockRMClient, times(7)).releaseAssignedContainer((ContainerId) any()); float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); Assert.assertEquals(progress, scheduler.getProgress(), 0); List<NodeReport> mockUpdatedNodes = mock(List.class); scheduler.onNodesUpdated(mockUpdatedNodes); drainableAppCallback.drain(); verify(mockApp).nodesUpdated(mockUpdatedNodes); Exception mockException = mock(Exception.class); scheduler.onError(mockException); drainableAppCallback.drain(); verify(mockApp).onError(mockException); scheduler.onShutdownRequest(); drainableAppCallback.drain(); verify(mockApp).appShutdownRequested(); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.stop(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); scheduler.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)//from w ww . jav a 2 s .c o m public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); Configuration conf = new Configuration(); scheduler.init(conf); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); scheduler.start(); String rack1 = "r1"; String rack2 = "r2"; String rack3 = "r3"; String node1Rack1 = "n1r1"; String node2Rack1 = "n2r1"; String node1Rack2 = "n1r2"; String node2Rack2 = "n2r2"; String node1Rack3 = "n1r3"; ApplicationAttemptId appId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0); ContainerId mockCId1 = ContainerId.newInstance(appId, 0); HeldContainer hc1 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc1.getNode()).thenReturn(node1Rack1); when(hc1.getRack()).thenReturn(rack1); when(hc1.getContainer().getId()).thenReturn(mockCId1); ContainerId mockCId2 = ContainerId.newInstance(appId, 1); HeldContainer hc2 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc2.getNode()).thenReturn(node2Rack1); when(hc2.getRack()).thenReturn(rack1); when(hc2.getContainer().getId()).thenReturn(mockCId2); ContainerId mockCId3 = ContainerId.newInstance(appId, 2); HeldContainer hc3 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc3.getNode()).thenReturn(node1Rack1); when(hc3.getRack()).thenReturn(rack1); when(hc3.getContainer().getId()).thenReturn(mockCId3); ContainerId mockCId4 = ContainerId.newInstance(appId, 3); HeldContainer hc4 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc4.getNode()).thenReturn(node2Rack1); when(hc4.getRack()).thenReturn(rack1); when(hc4.getContainer().getId()).thenReturn(mockCId4); ContainerId mockCId5 = ContainerId.newInstance(appId, 4); HeldContainer hc5 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc5.getNode()).thenReturn(node1Rack2); when(hc5.getRack()).thenReturn(rack2); when(hc5.getContainer().getId()).thenReturn(mockCId5); ContainerId mockCId6 = ContainerId.newInstance(appId, 5); HeldContainer hc6 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc6.getNode()).thenReturn(node2Rack2); when(hc6.getRack()).thenReturn(rack2); when(hc6.getContainer().getId()).thenReturn(mockCId6); ContainerId mockCId7 = ContainerId.newInstance(appId, 6); HeldContainer hc7 = mock(HeldContainer.class, RETURNS_DEEP_STUBS); when(hc7.getNode()).thenReturn(node1Rack3); when(hc7.getRack()).thenReturn(rack3); when(hc7.getContainer().getId()).thenReturn(mockCId7); scheduler.heldContainers.put(mockCId1, hc1); scheduler.heldContainers.put(mockCId2, hc2); scheduler.heldContainers.put(mockCId3, hc3); scheduler.heldContainers.put(mockCId4, hc4); scheduler.heldContainers.put(mockCId5, hc5); scheduler.heldContainers.put(mockCId6, hc6); scheduler.heldContainers.put(mockCId7, hc7); // test empty case scheduler.sessionNumMinHeldContainers = 0; scheduler.determineMinHeldContainers(); Assert.assertEquals(0, scheduler.sessionMinHeldContainers.size()); // test min >= held scheduler.sessionNumMinHeldContainers = 7; scheduler.determineMinHeldContainers(); Assert.assertEquals(7, scheduler.sessionMinHeldContainers.size()); // test min < held scheduler.sessionNumMinHeldContainers = 5; scheduler.determineMinHeldContainers(); Assert.assertEquals(5, scheduler.sessionMinHeldContainers.size()); Set<HeldContainer> heldContainers = Sets.newHashSet(); for (ContainerId cId : scheduler.sessionMinHeldContainers) { heldContainers.add(scheduler.heldContainers.get(cId)); } Set<String> racks = Sets.newHashSet(); Set<String> nodes = Sets.newHashSet(); for (HeldContainer hc : heldContainers) { nodes.add(hc.getNode()); racks.add(hc.getRack()); } // 1 container from each node in rack1 and rack2. 1 container from rack3. // covers not enough containers in rack (rack 3) // covers just enough containers in rack (rack 2) // covers more than enough containers in rack (rack 1) Assert.assertEquals(5, nodes.size()); Assert.assertTrue(nodes.contains(node1Rack1) && nodes.contains(node2Rack1) && nodes.contains(node1Rack2) && nodes.contains(node2Rack2) && nodes.contains(node1Rack3)); Assert.assertEquals(3, racks.size()); Assert.assertTrue(racks.contains(rack1) && racks.contains(rack2) && racks.contains(rack3)); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.stop(); scheduler.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from ww w. j a v a2 s . com*/ public void testTaskSchedulerRandomReuseExpireTime() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; TaskSchedulerWithDrainableAppCallback scheduler1 = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); TaskSchedulerWithDrainableAppCallback scheduler2 = new TaskSchedulerWithDrainableAppCallback(mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); long minTime = 1000l; long maxTime = 100000l; Configuration conf1 = new Configuration(); conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime); conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, minTime); scheduler1.init(conf1); Configuration conf2 = new Configuration(); conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime); conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, maxTime); scheduler2.init(conf2); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); Resource mockClusterResource = mock(Resource.class); when(mockRMClient.getAvailableResources()).thenReturn(mockClusterResource); scheduler1.start(); scheduler2.start(); // when min == max the expire time is always min for (int i = 0; i < 10; ++i) { Assert.assertEquals(minTime, scheduler1.getHeldContainerExpireTime(0)); } long lastExpireTime = 0; // when min < max the expire time is random in between min and max for (int i = 0; i < 10; ++i) { long currExpireTime = scheduler2.getHeldContainerExpireTime(0); Assert.assertTrue("min: " + minTime + " curr: " + currExpireTime + " max: " + maxTime, (minTime <= currExpireTime && currExpireTime <= maxTime)); Assert.assertNotEquals(lastExpireTime, currExpireTime); lastExpireTime = currExpireTime; } String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler1.stop(); scheduler1.close(); scheduler2.stop(); scheduler2.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 5000)//from ww w . j a v a 2 s . c o m public void testTaskSchedulerPreemption() throws Exception { RackResolver.init(new YarnConfiguration()); TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TezAMRMClientAsync<CookieContainerRequest> mockRMClient = mock(TezAMRMClientAsync.class); String appHost = "host"; int appPort = 0; String appUrl = "url"; final TaskSchedulerWithDrainableAppCallback scheduler = new TaskSchedulerWithDrainableAppCallback(mockApp, new PreemptionMatcher(), appHost, appPort, appUrl, mockRMClient, mockAppContext); TaskSchedulerAppCallbackDrainable drainableAppCallback = scheduler.getDrainableAppCallback(); Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); scheduler.init(conf); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); when(mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())) .thenReturn(mockRegResponse); scheduler.start(); Resource totalResource = Resource.newInstance(4000, 4); when(mockRMClient.getAvailableResources()).thenReturn(totalResource); // no preemption scheduler.getProgress(); drainableAppCallback.drain(); Assert.assertEquals(totalResource, scheduler.getTotalResources()); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // allocate task Object mockTask1 = mock(Object.class); Object mockTask2 = mock(Object.class); Object mockTask3 = mock(Object.class); Object mockTask3Wait = mock(Object.class); Object mockTask3Retry = mock(Object.class); Object mockTask3KillA = mock(Object.class); Object mockTask3KillB = mock(Object.class); Object obj3 = new Object(); Priority pri2 = Priority.newInstance(2); Priority pri4 = Priority.newInstance(4); Priority pri5 = Priority.newInstance(5); Priority pri6 = Priority.newInstance(6); ArgumentCaptor<CookieContainerRequest> requestCaptor = ArgumentCaptor .forClass(CookieContainerRequest.class); final ArrayList<CookieContainerRequest> anyContainers = new ArrayList<CookieContainerRequest>(); Resource taskAsk = Resource.newInstance(1024, 1); scheduler.allocateTask(mockTask1, taskAsk, null, null, pri2, null, null); drainableAppCallback.drain(); verify(mockRMClient, times(1)).addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); scheduler.allocateTask(mockTask3, taskAsk, null, null, pri6, obj3, null); drainableAppCallback.drain(); verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); // later one in the allocation gets killed between the two task3's scheduler.allocateTask(mockTask3KillA, taskAsk, null, null, pri6, obj3, null); drainableAppCallback.drain(); verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); // later one in the allocation gets killed between the two task3's scheduler.allocateTask(mockTask3KillB, taskAsk, null, null, pri6, obj3, null); drainableAppCallback.drain(); verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); Resource freeResource = Resource.newInstance(500, 0); when(mockRMClient.getAvailableResources()).thenReturn(freeResource); scheduler.getProgress(); drainableAppCallback.drain(); Assert.assertEquals(totalResource, scheduler.getTotalResources()); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); final List<ArrayList<CookieContainerRequest>> anyList = new LinkedList<ArrayList<CookieContainerRequest>>(); final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>(); anyList.add(anyContainers); List<Container> containers = new ArrayList<Container>(); Container mockContainer1 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer1.getNodeId().getHost()).thenReturn("host1"); when(mockContainer1.getResource()).thenReturn(taskAsk); when(mockContainer1.getPriority()).thenReturn(pri2); ContainerId mockCId1 = mock(ContainerId.class); when(mockContainer1.getId()).thenReturn(mockCId1); containers.add(mockContainer1); Container mockContainer2 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer2.getNodeId().getHost()).thenReturn("host1"); when(mockContainer2.getResource()).thenReturn(taskAsk); when(mockContainer2.getPriority()).thenReturn(pri6); ContainerId mockCId2 = mock(ContainerId.class); when(mockContainer2.getId()).thenReturn(mockCId2); containers.add(mockContainer2); Container mockContainer3A = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer3A.getNodeId().getHost()).thenReturn("host1"); when(mockContainer3A.getResource()).thenReturn(taskAsk); when(mockContainer3A.getPriority()).thenReturn(pri6); ContainerId mockCId3A = mock(ContainerId.class); when(mockContainer3A.getId()).thenReturn(mockCId3A); containers.add(mockContainer3A); Container mockContainer3B = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer3B.getNodeId().getHost()).thenReturn("host1"); when(mockContainer3B.getResource()).thenReturn(taskAsk); when(mockContainer3B.getPriority()).thenReturn(pri2); // high priority container ContainerId mockCId3B = mock(ContainerId.class); when(mockContainer3B.getId()).thenReturn(mockCId3B); containers.add(mockContainer3B); when(mockRMClient.getMatchingRequests((Priority) any(), eq("host1"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); // RackResolver by default puts hosts in default-rack when(mockRMClient.getMatchingRequests((Priority) any(), eq("/default-rack"), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { return emptyList; } }); when(mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { int calls = 0; @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { if (calls > 0) { anyContainers.remove(0); } calls++; return anyList; } }); Mockito.doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { Object[] args = invocation.getArguments(); ContainerId cId = (ContainerId) args[0]; scheduler.deallocateContainer(cId); return null; } }).when(mockApp).preemptContainer((ContainerId) any()); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); Assert.assertEquals(4, scheduler.taskAllocations.size()); Assert.assertEquals(4096, scheduler.allocatedResources.getMemory()); Assert.assertEquals(mockCId1, scheduler.taskAllocations.get(mockTask1).getId()); Assert.assertEquals(mockCId2, scheduler.taskAllocations.get(mockTask3).getId()); Assert.assertEquals(mockCId3A, scheduler.taskAllocations.get(mockTask3KillA).getId()); // high priority container assigned to lower pri task. This task should still be preempted // because the task priority is relevant for preemption and not the container priority Assert.assertEquals(mockCId3B, scheduler.taskAllocations.get(mockTask3KillB).getId()); // no preemption scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); Object mockTask3WaitCookie = new Object(); scheduler.allocateTask(mockTask3Wait, taskAsk, null, null, pri6, obj3, mockTask3WaitCookie); // no preemption - same pri scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); Priority pri8 = Priority.newInstance(8); Container mockContainer4 = mock(Container.class, RETURNS_DEEP_STUBS); when(mockContainer4.getNodeId().getHost()).thenReturn("host1"); when(mockContainer4.getResource()).thenReturn(taskAsk); when(mockContainer4.getPriority()).thenReturn(pri8); ContainerId mockCId4 = mock(ContainerId.class); when(mockContainer4.getId()).thenReturn(mockCId4); containers.clear(); containers.add(mockContainer4); // Fudge new container being present in delayed allocation list due to race HeldContainer heldContainer = new HeldContainer(mockContainer4, -1, -1, null); scheduler.delayedContainerManager.delayedContainers.add(heldContainer); // no preemption - container assignment attempts < 3 scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); heldContainer.incrementAssignmentAttempts(); // no preemption - container assignment attempts < 3 scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); heldContainer.incrementAssignmentAttempts(); heldContainer.incrementAssignmentAttempts(); // preemption - container released and resource asked again scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId4); verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture()); CookieContainerRequest reAdded = requestCaptor.getValue(); Assert.assertEquals(pri6, reAdded.getPriority()); Assert.assertEquals(taskAsk, reAdded.getCapability()); Assert.assertEquals(mockTask3WaitCookie, reAdded.getCookie().getAppCookie()); // remove fudging. scheduler.delayedContainerManager.delayedContainers.clear(); scheduler.allocateTask(mockTask3Retry, taskAsk, null, null, pri5, obj3, null); // no preemption - higher pri. exact match scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); for (int i = 0; i < 11; ++i) { scheduler.allocateTask(mockTask2, taskAsk, null, null, pri4, null, null); } drainableAppCallback.drain(); // mockTaskPri3KillB gets preempted to clear 10% of outstanding running preemptable tasks // this is also a higher priority container than the pending task priority but was running a // lower priority task. Task priority is relevant for preemption and not container priority as // containers can run tasks of different priorities scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3B); // next 3 heartbeats do nothing, waiting for the RM to act on the last released resources scheduler.getProgress(); scheduler.getProgress(); scheduler.getProgress(); verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any()); scheduler.getProgress(); drainableAppCallback.drain(); // Next oldest mockTaskPri3KillA gets preempted to clear 10% of outstanding running preemptable tasks verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3A); AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, "", appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.stop(); drainableAppCallback.drain(); scheduler.close(); }
From source file:org.apache.tez.dag.app.rm.TestTaskScheduler.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from w ww. j a v a2 s . c o m*/ public void testLocalityMatching() throws Exception { RackResolver.init(new Configuration()); TaskSchedulerAppCallback appClient = mock(TaskSchedulerAppCallback.class); TezAMRMClientAsync<CookieContainerRequest> amrmClient = mock(TezAMRMClientAsync.class); AppContext mockAppContext = mock(AppContext.class); when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING); TaskSchedulerWithDrainableAppCallback taskScheduler = new TaskSchedulerWithDrainableAppCallback(appClient, new AlwaysMatchesContainerMatcher(), "host", 0, "", amrmClient, mockAppContext); TaskSchedulerAppCallbackDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback(); Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); taskScheduler.init(conf); RegisterApplicationMasterResponse mockRegResponse = mock(RegisterApplicationMasterResponse.class); Resource mockMaxResource = mock(Resource.class); Map<ApplicationAccessType, String> mockAcls = mock(Map.class); when(mockRegResponse.getMaximumResourceCapability()).thenReturn(mockMaxResource); when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls); when(amrmClient.registerApplicationMaster(anyString(), anyInt(), anyString())).thenReturn(mockRegResponse); taskScheduler.start(); Resource resource = Resource.newInstance(1024, 1); Priority priority = Priority.newInstance(1); String hostsTask1[] = { "host1" }; String hostsTask2[] = { "non-allocated-host" }; String defaultRack[] = { "/default-rack" }; String otherRack[] = { "/other-rack" }; Object mockTask1 = mock(Object.class); CookieContainerRequest mockCookie1 = mock(CookieContainerRequest.class, RETURNS_DEEP_STUBS); when(mockCookie1.getCookie().getTask()).thenReturn(mockTask1); Object mockTask2 = mock(Object.class); CookieContainerRequest mockCookie2 = mock(CookieContainerRequest.class, RETURNS_DEEP_STUBS); when(mockCookie2.getCookie().getTask()).thenReturn(mockTask2); Container containerHost1 = createContainer(1, "host1", resource, priority); Container containerHost3 = createContainer(2, "host3", resource, priority); List<Container> allocatedContainers = new LinkedList<Container>(); allocatedContainers.add(containerHost3); allocatedContainers.add(containerHost1); final Map<String, List<CookieContainerRequest>> matchingMap = new HashMap<String, List<CookieContainerRequest>>(); taskScheduler.allocateTask(mockTask1, resource, hostsTask1, defaultRack, priority, null, mockCookie1); drainableAppCallback.drain(); List<CookieContainerRequest> host1List = new ArrayList<CookieContainerRequest>(); host1List.add(mockCookie1); List<CookieContainerRequest> defaultRackList = new ArrayList<CookieContainerRequest>(); defaultRackList.add(mockCookie1); matchingMap.put(hostsTask1[0], host1List); matchingMap.put(defaultRack[0], defaultRackList); List<CookieContainerRequest> nonAllocatedHostList = new ArrayList<YarnTaskSchedulerService.CookieContainerRequest>(); nonAllocatedHostList.add(mockCookie2); List<CookieContainerRequest> otherRackList = new ArrayList<YarnTaskSchedulerService.CookieContainerRequest>(); otherRackList.add(mockCookie2); taskScheduler.allocateTask(mockTask2, resource, hostsTask2, otherRack, priority, null, mockCookie2); drainableAppCallback.drain(); matchingMap.put(hostsTask2[0], nonAllocatedHostList); matchingMap.put(otherRack[0], otherRackList); List<CookieContainerRequest> anyList = new LinkedList<YarnTaskSchedulerService.CookieContainerRequest>(); anyList.add(mockCookie1); anyList.add(mockCookie2); matchingMap.put(ResourceRequest.ANY, anyList); final List<ArrayList<CookieContainerRequest>> emptyList = new LinkedList<ArrayList<CookieContainerRequest>>(); when(amrmClient.getMatchingRequests((Priority) any(), anyString(), (Resource) any())) .thenAnswer(new Answer<List<? extends Collection<CookieContainerRequest>>>() { @Override public List<? extends Collection<CookieContainerRequest>> answer(InvocationOnMock invocation) throws Throwable { String location = (String) invocation.getArguments()[1]; if (matchingMap.get(location) != null) { CookieContainerRequest matched = matchingMap.get(location).get(0); // Remove matched from matchingMap - assuming TaskScheduler will // pick the first entry. Iterator<Entry<String, List<CookieContainerRequest>>> iter = matchingMap.entrySet() .iterator(); while (iter.hasNext()) { Entry<String, List<CookieContainerRequest>> entry = iter.next(); if (entry.getValue().remove(matched)) { if (entry.getValue().size() == 0) { iter.remove(); } } } return Collections.singletonList(Collections.singletonList(matched)); } else { return emptyList; } } }); taskScheduler.onContainersAllocated(allocatedContainers); drainableAppCallback.drain(); ArgumentCaptor<Object> taskCaptor = ArgumentCaptor.forClass(Object.class); ArgumentCaptor<Container> containerCaptor = ArgumentCaptor.forClass(Container.class); verify(appClient, times(2)).taskAllocated(taskCaptor.capture(), any(), containerCaptor.capture()); // Expected containerHost1 allocated to task1 due to locality, // containerHost3 allocated to task2. List<Container> assignedContainers = containerCaptor.getAllValues(); int container1Pos = assignedContainers.indexOf(containerHost1); assertTrue("Container: " + containerHost1 + " was not assigned", container1Pos != -1); assertEquals("Task 1 was not allocated to containerHost1", mockTask1, taskCaptor.getAllValues().get(container1Pos)); int container2Pos = assignedContainers.indexOf(containerHost3); assertTrue("Container: " + containerHost3 + " was not assigned", container2Pos != -1); assertEquals("Task 2 was not allocated to containerHost3", mockTask2, taskCaptor.getAllValues().get(container2Pos)); AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, "", ""); when(appClient.getFinalAppStatus()).thenReturn(finalStatus); taskScheduler.close(); }
From source file:org.apache.tez.hadoop.shim.TestHadoopShim28.java
License:Apache License
@Test public void testApplyFinalApplicationStatusCorrection() { HadoopShim shim = new HadoopShim28(); // Session mode success/failure, change to ended Assert.assertEquals(FinalApplicationStatus.ENDED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.SUCCEEDED, true, false)); Assert.assertEquals(FinalApplicationStatus.ENDED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, true, false)); // Non-session mode success/failure, retain success/failure Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.SUCCEEDED, false, false)); Assert.assertEquals(FinalApplicationStatus.FAILED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, false, false)); // Session and non-session mode error, retain failed. Assert.assertEquals(FinalApplicationStatus.FAILED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, true, true)); Assert.assertEquals(FinalApplicationStatus.FAILED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, false, true)); // Session and non-session mode killed is killed. Assert.assertEquals(FinalApplicationStatus.KILLED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.KILLED, true, false)); Assert.assertEquals(FinalApplicationStatus.KILLED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.KILLED, false, false)); // Session and non-session mode undefined is undefined. Assert.assertEquals(FinalApplicationStatus.UNDEFINED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.UNDEFINED, true, false)); Assert.assertEquals(FinalApplicationStatus.UNDEFINED, shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.UNDEFINED, false, false)); }
From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java
License:Apache License
private void stopAndVerifyYarnApp(TezClient tezSession) throws TezException, IOException, YarnException { ApplicationId appId = tezSession.getAppMasterApplicationId(); tezSession.stop();/*from w w w. j a v a 2 s . com*/ Assert.assertEquals(TezAppMasterStatus.SHUTDOWN, tezSession.getAppMasterStatus()); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState().equals(YarnApplicationState.FINISHED) || appReport.getYarnApplicationState().equals(YarnApplicationState.FAILED) || appReport.getYarnApplicationState().equals(YarnApplicationState.KILLED)) { break; } } ApplicationReport appReport = yarnClient.getApplicationReport(appId); Assert.assertEquals(YarnApplicationState.FINISHED, appReport.getYarnApplicationState()); Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus()); }
From source file:org.apache.tez.mapreduce.TestMRRJobsDAGApi.java
License:Apache License
public State testMRRSleepJobDagSubmitCore(boolean dagViaRPC, boolean killDagWhileRunning, boolean closeSessionBeforeSubmit, TezClient reUseTezSession, boolean genSplitsInAM, Class<? extends InputInitializer> initializerClass, Map<String, LocalResource> additionalLocalResources) throws IOException, InterruptedException, TezException, ClassNotFoundException, YarnException { LOG.info("\n\n\nStarting testMRRSleepJobDagSubmit()."); JobConf stage1Conf = new JobConf(mrrTezCluster.getConfig()); JobConf stage2Conf = new JobConf(mrrTezCluster.getConfig()); JobConf stage3Conf = new JobConf(mrrTezCluster.getConfig()); stage1Conf.setLong(MRRSleepJob.MAP_SLEEP_TIME, 1); stage1Conf.setInt(MRRSleepJob.MAP_SLEEP_COUNT, 1); stage1Conf.setInt(MRJobConfig.NUM_MAPS, 1); stage1Conf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName()); stage1Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); stage1Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName()); stage1Conf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, SleepInputFormat.class.getName()); stage1Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName()); stage2Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1); stage2Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1); stage2Conf.setInt(MRJobConfig.NUM_REDUCES, 1); stage2Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName()); stage2Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); stage2Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName()); stage2Conf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName()); stage3Conf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, 1); stage3Conf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, 1); stage3Conf.setInt(MRJobConfig.NUM_REDUCES, 1); stage3Conf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName()); stage3Conf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); stage3Conf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName()); MRHelpers.translateMRConfToTez(stage1Conf); MRHelpers.translateMRConfToTez(stage2Conf); MRHelpers.translateMRConfToTez(stage3Conf); MRHelpers.configureMRApiUsage(stage1Conf); MRHelpers.configureMRApiUsage(stage2Conf); MRHelpers.configureMRApiUsage(stage3Conf); Path remoteStagingDir = remoteFs .makeQualified(new Path("/tmp", String.valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf); UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf); DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000)); Class<? extends InputInitializer> inputInitializerClazz = genSplitsInAM ? (initializerClass == null ? MRInputAMSplitGenerator.class : initializerClass) : null;/*from ww w.ja v a2 s . c om*/ LOG.info("Using initializer class: " + initializerClass); DataSourceDescriptor dsd; if (!genSplitsInAM) { dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(stage1Conf, remoteStagingDir, true); } else { if (initializerClass == null) { dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).build(); } else { InputInitializerDescriptor iid = InputInitializerDescriptor.create(inputInitializerClazz.getName()); dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class) .setCustomInitializerDescriptor(iid).build(); } } Vertex stage1Vertex = Vertex.create("map", ProcessorDescriptor.create(MapProcessor.class.getName()).setUserPayload(stage1Payload), dsd.getNumberOfShards(), Resource.newInstance(256, 1)); stage1Vertex.addDataSource("MRInput", dsd); Vertex stage2Vertex = Vertex.create("ireduce", ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage2Payload), 1, Resource.newInstance(256, 1)); Vertex stage3Vertex = Vertex.create("reduce", ProcessorDescriptor.create(ReduceProcessor.class.getName()).setUserPayload(stage3Payload), 1, Resource.newInstance(256, 1)); stage3Conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, true); DataSinkDescriptor dataSinkDescriptor = MROutputLegacy .createConfigBuilder(stage3Conf, NullOutputFormat.class).build(); Assert.assertFalse(dataSinkDescriptor.getOutputDescriptor().getHistoryText().isEmpty()); stage3Vertex.addDataSink("MROutput", dataSinkDescriptor); // TODO env, resources dag.addVertex(stage1Vertex); dag.addVertex(stage2Vertex); dag.addVertex(stage3Vertex); Edge edge1 = Edge.create(stage1Vertex, stage2Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload), InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload))); Edge edge2 = Edge.create(stage2Vertex, stage3Vertex, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload), InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload))); dag.addEdge(edge1); dag.addEdge(edge2); TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); DAGClient dagClient = null; boolean reuseSession = reUseTezSession != null; TezClient tezSession = null; if (!dagViaRPC) { Preconditions.checkArgument(reuseSession == false); } if (!reuseSession) { TezConfiguration tempTezconf = new TezConfiguration(tezConf); if (!dagViaRPC) { tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); } else { tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); } tezSession = TezClient.create("testsession", tempTezconf); tezSession.start(); } else { tezSession = reUseTezSession; } if (!dagViaRPC) { // TODO Use utility method post TEZ-205 to figure out AM arguments etc. dagClient = tezSession.submitDAG(dag); } if (dagViaRPC && closeSessionBeforeSubmit) { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); boolean sentKillSession = false; while (true) { Thread.sleep(500l); ApplicationReport appReport = yarnClient .getApplicationReport(tezSession.getAppMasterApplicationId()); if (appReport == null) { continue; } YarnApplicationState appState = appReport.getYarnApplicationState(); if (!sentKillSession) { if (appState == YarnApplicationState.RUNNING) { tezSession.stop(); sentKillSession = true; } } else { if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) { LOG.info("Application completed after sending session shutdown" + ", yarnApplicationState=" + appState + ", finalAppStatus=" + appReport.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, appState); Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus()); break; } } } yarnClient.stop(); return null; } if (dagViaRPC) { LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId() + " and Dag Name=" + dag.getName()); if (additionalLocalResources != null) { tezSession.addAppMasterLocalFiles(additionalLocalResources); } dagClient = tezSession.submitDAG(dag); Assert.assertEquals(TezAppMasterStatus.RUNNING, tezSession.getAppMasterStatus()); } DAGStatus dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { LOG.info( "Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState()); Thread.sleep(500l); if (killDagWhileRunning && dagStatus.getState() == DAGStatus.State.RUNNING) { LOG.info("Killing running dag/session"); if (dagViaRPC) { tezSession.stop(); } else { dagClient.tryKillDAG(); } } dagStatus = dagClient.getDAGStatus(null); } if (!reuseSession) { tezSession.stop(); } return dagStatus.getState(); }