List of usage examples for org.apache.hadoop.yarn.api.records NodeId newInstance
@Public @Stable public static NodeId newInstance(String host, int port)
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testSimpleReuseAnyMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from www . j a va2s . com drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host2"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, "host4", "/rack4"); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, "host1"); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, "host6", "/rack6"); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); TaskRequest reqv0t1 = taskRequestCaptor.scheduleTask(taskv0t1); TaskRequest reqv0t2 = taskRequestCaptor.scheduleTask(taskv0t2); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // finish v0t0 successfully, verify v0t1 is skipped and v1t0 assigned instead // since host locality is preferred to rack locality and lower priority vertex // is not blocked by higher priority vertex assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // finish v1t0 successfully, verify v0t1 is assigned assertTrue(scheduler.deallocateTask(taskv1t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t1.task, taskv0t1.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t1); // finish v0t1 successfully, verify v0t2 is assigned assertTrue(scheduler.deallocateTask(taskv0t1.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t2.task, taskv0t2.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t2); // fail v0t2 and verify container is released instead of reused for v1t1 assertTrue(scheduler.deallocateTask(taskv0t2.task, false, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testReuseWithAffinity() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from w w w. ja v a2s .c om drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, hostsv0t0); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); taskRequestCaptor.scheduleTask(taskv0t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // add a new request for this container MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, "host1"); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0, cid1); // finish v0t0 successfully, verify v0t1 is skipped even though it is node-local // and v1t0 assigned instead for affinity assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testReuseVertexDescendants() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); // vertex 0 and vertex 2 are root vertices and vertex 1 is a child of vertex 0 DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(3); when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[] { 0x2 })); when(mockDagInfo.getVertexDescendants(1)).thenReturn(new BitSet()); when(mockDagInfo.getVertexDescendants(2)).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from w ww. j av a 2 s . co m drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); Priority priorityv2 = Priority.newInstance(3); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t0.task)).thenReturn(0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host3"); when(mockApp.getVertexIndexForTask(taskv0t1.task)).thenReturn(0); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv1t0.task)).thenReturn(1); MockTaskInfo taskv2t0 = new MockTaskInfo("taskv2t0", priorityv2, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv2t0.task)).thenReturn(2); MockTaskInfo taskv2t1 = new MockTaskInfo("taskv2t1", priorityv2, "host3"); when(mockApp.getVertexIndexForTask(taskv2t1.task)).thenReturn(2); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); TaskRequest reqv0t1 = taskRequestCaptor.scheduleTask(taskv0t1); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); TaskRequest reqv2t0 = taskRequestCaptor.scheduleTask(taskv2t0); taskRequestCaptor.scheduleTask(taskv2t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // finish v0t0 successfully, verify v1t0 is skipped and v2t0 assigned instead // since host locality beats rack locality for unblocked vertex v2 and // v1 is blocked by pending v0 request assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv2t0.task, taskv2t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv2t0); // finish v2t0 successfully, verify v0t1 is assigned since it is higher // priority than v2 assertTrue(scheduler.deallocateTask(taskv2t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t1.task, taskv0t1.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t1); // finish v2t0 successfully, verify v1t0 is assigned since it is now unblocked assertTrue(scheduler.deallocateTask(taskv0t1.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // fail v1t0 and verify container is released instead of reused for v2t1 assertTrue(scheduler.deallocateTask(taskv1t0.task, false, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testSessionContainers() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 4000); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 5000); conf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, 5); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); when(mockApp.isSession()).thenReturn(true); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();/*from ww w .j a va2 s. c om*/ drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); final String rack1 = "/r1"; final String rack2 = "/r2"; final String rack3 = "/r3"; final String node1Rack1 = "n1r1"; final String node2Rack1 = "n2r1"; final String node1Rack2 = "n1r2"; final String node2Rack2 = "n2r2"; final String node1Rack3 = "n1r3"; MockDNSToSwitchMapping.addRackMapping(node1Rack1, rack1); MockDNSToSwitchMapping.addRackMapping(node2Rack1, rack1); MockDNSToSwitchMapping.addRackMapping(node1Rack2, rack2); MockDNSToSwitchMapping.addRackMapping(node2Rack2, rack2); MockDNSToSwitchMapping.addRackMapping(node1Rack3, rack3); Priority priorityv0 = Priority.newInstance(1); MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, node1Rack1, rack1); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, node2Rack1, rack1); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, node1Rack1, rack1); MockTaskInfo taskv0t3 = new MockTaskInfo("taskv0t3", priorityv0, node2Rack1, rack1); MockTaskInfo taskv0t4 = new MockTaskInfo("taskv0t4", priorityv0, node1Rack2, rack2); MockTaskInfo taskv0t5 = new MockTaskInfo("taskv0t5", priorityv0, node2Rack2, rack2); MockTaskInfo taskv0t6 = new MockTaskInfo("taskv0t6", priorityv0, node1Rack3, rack3); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); TaskRequest reqv0t1 = taskRequestCaptor.scheduleTask(taskv0t1); TaskRequest reqv0t2 = taskRequestCaptor.scheduleTask(taskv0t2); TaskRequest reqv0t3 = taskRequestCaptor.scheduleTask(taskv0t3); TaskRequest reqv0t4 = taskRequestCaptor.scheduleTask(taskv0t4); TaskRequest reqv0t5 = taskRequestCaptor.scheduleTask(taskv0t5); TaskRequest reqv0t6 = taskRequestCaptor.scheduleTask(taskv0t6); List<Container> containers = new ArrayList<>(); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); NodeId n1r1 = NodeId.newInstance(node1Rack1, 1); Container container1 = Container.newInstance(cid1, n1r1, null, taskv0t0.capability, priorityv0, null); containers.add(container1); ContainerId cid2 = ContainerId.newContainerId(attemptId, 2); NodeId n2r1 = NodeId.newInstance(node2Rack1, 1); Container container2 = Container.newInstance(cid2, n2r1, null, taskv0t1.capability, priorityv0, null); containers.add(container2); ContainerId cid3 = ContainerId.newContainerId(attemptId, 3); Container container3 = Container.newInstance(cid3, n1r1, null, taskv0t2.capability, priorityv0, null); containers.add(container3); ContainerId cid4 = ContainerId.newContainerId(attemptId, 4); Container container4 = Container.newInstance(cid4, n2r1, null, taskv0t3.capability, priorityv0, null); containers.add(container4); ContainerId cid5 = ContainerId.newContainerId(attemptId, 5); NodeId n1r2 = NodeId.newInstance(node1Rack2, 1); Container container5 = Container.newInstance(cid5, n1r2, null, taskv0t4.capability, priorityv0, null); containers.add(container5); ContainerId cid6 = ContainerId.newContainerId(attemptId, 6); NodeId n2r2 = NodeId.newInstance(node2Rack2, 1); Container container6 = Container.newInstance(cid6, n2r2, null, taskv0t5.capability, priorityv0, null); containers.add(container6); ContainerId cid7 = ContainerId.newContainerId(attemptId, 7); NodeId n1r3 = NodeId.newInstance(node1Rack3, 1); Container container7 = Container.newInstance(cid7, n1r3, null, taskv0t6.capability, priorityv0, null); containers.add(container7); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); verify(mockApp).taskAllocated(taskv0t1.task, taskv0t1.cookie, container2); verify(mockRMClient).removeContainerRequest(reqv0t1); verify(mockApp).taskAllocated(taskv0t2.task, taskv0t2.cookie, container3); verify(mockRMClient).removeContainerRequest(reqv0t2); verify(mockApp).taskAllocated(taskv0t3.task, taskv0t3.cookie, container4); verify(mockRMClient).removeContainerRequest(reqv0t3); verify(mockApp).taskAllocated(taskv0t4.task, taskv0t4.cookie, container5); verify(mockRMClient).removeContainerRequest(reqv0t4); verify(mockApp).taskAllocated(taskv0t5.task, taskv0t5.cookie, container6); verify(mockRMClient).removeContainerRequest(reqv0t5); verify(mockApp).taskAllocated(taskv0t6.task, taskv0t6.cookie, container7); verify(mockRMClient).removeContainerRequest(reqv0t6); clock.incrementTime(10000); drainableAppCallback.drain(); assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); assertTrue(scheduler.deallocateTask(taskv0t1.task, true, null, null)); assertTrue(scheduler.deallocateTask(taskv0t2.task, true, null, null)); assertTrue(scheduler.deallocateTask(taskv0t3.task, true, null, null)); assertTrue(scheduler.deallocateTask(taskv0t4.task, true, null, null)); assertTrue(scheduler.deallocateTask(taskv0t5.task, true, null, null)); assertTrue(scheduler.deallocateTask(taskv0t6.task, true, null, null)); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); // verify only two of the containers were released after idle expiration // and the rest were spread across the nodes and racks clock.incrementTime(5000); drainableAppCallback.drain(); verify(mockApp, times(2)).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, times(2)).releaseAssignedContainer(any(ContainerId.class)); Set<String> hosts = new HashSet<>(); Set<String> racks = new HashSet<>(); for (HeldContainer hc : scheduler.getSessionContainers()) { hosts.add(hc.getHost()); racks.add(hc.getRack()); } assertEquals(5, hosts.size()); assertEquals(3, racks.size()); assertTrue(hosts.contains(node1Rack1)); assertTrue(hosts.contains(node2Rack1)); assertTrue(hosts.contains(node1Rack2)); assertTrue(hosts.contains(node2Rack2)); assertTrue(hosts.contains(node1Rack3)); assertTrue(racks.contains(rack1)); assertTrue(racks.contains(rack2)); assertTrue(racks.contains(rack3)); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 50000) public void testPreemptionNoHeadroom() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE, 10); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 3); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, 60 * 1000); // vertex 0 and vertex 2 are root vertices and vertex 1 is a child of vertex 0 DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(3); when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[] { 0x2 })); when(mockDagInfo.getVertexDescendants(1)).thenReturn(new BitSet()); when(mockDagInfo.getVertexDescendants(2)).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from w ww .j a v a2s.c o m drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); Priority priorityv2 = Priority.newInstance(3); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t0.task)).thenReturn(0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t1.task)).thenReturn(0); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv1t0.task)).thenReturn(1); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv1t1.task)).thenReturn(1); MockTaskInfo taskv2t0 = new MockTaskInfo("taskv2t0", priorityv2, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv2t0.task)).thenReturn(2); // asks for two tasks for vertex 1 and start running one of them TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); TaskRequest reqv1t1 = taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv1t0.capability, priorityv1, null); scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // start running the other task for vertex 1 a bit later clock.incrementTime(1000); ContainerId cid2 = ContainerId.newContainerId(attemptId, 2); Container container2 = Container.newInstance(cid2, host1, null, taskv1t0.capability, priorityv1, null); scheduler.onContainersAllocated(Collections.singletonList(container2)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv1t1.task, taskv1t1.cookie, container2); verify(mockRMClient).removeContainerRequest(reqv1t1); // add a request for vertex 0 but there is no headroom when(mockRMClient.getAvailableResources()).thenReturn(Resources.none()); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); // should preempt after enough heartbeats to get past preemption interval // only the youngest container should be preempted to meet the demand scheduler.getProgress(); scheduler.getProgress(); scheduler.getProgress(); drainableAppCallback.drain(); verify(mockApp, times(1)).preemptContainer(any(ContainerId.class)); verify(mockApp).preemptContainer(cid2); assertEquals(taskv1t1.task, scheduler.deallocateContainer(cid2)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid2); verify(mockRMClient).releaseAssignedContainer(cid2); verify(mockApp, never()).containerBeingReleased(cid1); verify(mockRMClient, never()).releaseAssignedContainer(cid1); // add a request for vertex 2 and allocate another container clock.incrementTime(1000); taskRequestCaptor.scheduleTask(taskv2t0); ContainerId cid3 = ContainerId.newContainerId(attemptId, 3); Container container3 = Container.newInstance(cid3, host1, null, taskv0t0.capability, priorityv0, null); scheduler.onContainersAllocated(Collections.singletonList(container3)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container3); verify(mockRMClient).removeContainerRequest(reqv0t0); // no more preemptions since v1 is not a descendant of v2 scheduler.getProgress(); scheduler.getProgress(); scheduler.getProgress(); drainableAppCallback.drain(); verify(mockApp, times(1)).preemptContainer(any(ContainerId.class)); // adding request for v0 should trigger preemption on next heartbeat taskRequestCaptor.scheduleTask(taskv0t1); scheduler.getProgress(); drainableAppCallback.drain(); verify(mockApp, times(2)).preemptContainer(any(ContainerId.class)); verify(mockApp).preemptContainer(cid1); assertEquals(taskv1t0.task, scheduler.deallocateContainer(cid1)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 50000) public void testContainerAssignmentReleaseNewContainers() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 4000); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 5000); conf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, 5); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); when(mockApp.isSession()).thenReturn(true); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from w ww .j a va 2 s .c o m drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); final String rack1 = "/r1"; final String rack2 = "/r2"; final String node1Rack1 = "n1r1"; final String node2Rack1 = "n2r1"; final String node1Rack2 = "n1r2"; MockDNSToSwitchMapping.addRackMapping(node1Rack1, rack1); MockDNSToSwitchMapping.addRackMapping(node2Rack1, rack1); MockDNSToSwitchMapping.addRackMapping(node1Rack2, rack2); Priority priorityv0 = Priority.newInstance(1); MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, node1Rack1, rack1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); NodeId n2r1 = NodeId.newInstance(node2Rack1, 1); Container container1 = Container.newInstance(cid1, n2r1, null, taskv0t0.capability, priorityv0, null); // verify new container is released is not immediately allocated scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); // app is not notified of the container being released since it never launched verify(mockApp, never()).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(eq(cid1)); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 50000) public void testIdleContainerAssignmentReuseNewContainers() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 4000); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 5000); conf.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, 5); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); when(mockApp.isSession()).thenReturn(true); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();// w w w . j a v a 2 s. c o m drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); final String rack1 = "/r1"; final String rack2 = "/r2"; final String node1Rack1 = "n1r1"; final String node2Rack1 = "n2r1"; final String node1Rack2 = "n1r2"; MockDNSToSwitchMapping.addRackMapping(node1Rack1, rack1); MockDNSToSwitchMapping.addRackMapping(node2Rack1, rack1); MockDNSToSwitchMapping.addRackMapping(node1Rack2, rack2); Priority priorityv0 = Priority.newInstance(1); MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, node1Rack1, rack1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); NodeId n2r1 = NodeId.newInstance(node2Rack1, 1); Container container1 = Container.newInstance(cid1, n2r1, null, taskv0t0.capability, priorityv0, null); // verify idle container is kept for now scheduler.onContainersAllocated(Collections.singletonList(container1)); clock.incrementTime(2000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(cid1); verify(mockRMClient, never()).releaseAssignedContainer(cid1); // verify idle container is released without being assigned to a task because rack-local reuse is // disabled TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient, never()).removeContainerRequest(reqv0t0); verify(mockApp, never()).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); // cancel the task request assertFalse(scheduler.deallocateTask(taskv0t0.task, false, null, null)); // allocate another container that's node-local ContainerId cid2 = ContainerId.newContainerId(attemptId, 2); NodeId n1r1 = NodeId.newInstance(node1Rack1, 1); Container container2 = Container.newInstance(cid2, n1r1, null, taskv0t0.capability, priorityv0, null); scheduler.onContainersAllocated(Collections.singletonList(container2)); clock.incrementTime(2000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(cid2); verify(mockRMClient, never()).releaseAssignedContainer(cid2); // reschedule the task, verify it's now scheduled without a container request // since node-local idle container is available reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0, false); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container2); verify(mockRMClient).removeContainerRequest(reqv0t0); }
From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager.java
License:Apache License
@Test(timeout = 5000) public void testEventRouting() throws Exception { AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS); NodeId nodeId = NodeId.newInstance("host1", 3131); when(appContext.getAllContainers().get(any(ContainerId.class)).getContainer().getNodeId()) .thenReturn(nodeId);//w ww. ja v a2 s . co m TaskHeartbeatHandler thh = mock(TaskHeartbeatHandler.class); ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class); Configuration conf = new Configuration(false); conf.set("testkey", "testvalue"); UserPayload defaultPayload = TezUtils.createUserPayloadFromConf(conf); String customTaskCommName = "customTaskComm"; List<NamedEntityDescriptor> taskCommDescriptors = new LinkedList<>(); ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(0, 3); UserPayload customPayload = UserPayload.create(bb); taskCommDescriptors.add(new NamedEntityDescriptor(customTaskCommName, FakeTaskComm.class.getName()) .setUserPayload(customPayload)); taskCommDescriptors.add(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(defaultPayload)); TaskCommManagerForMultipleCommTest tcm = new TaskCommManagerForMultipleCommTest(appContext, thh, chh, taskCommDescriptors); try { tcm.init(new Configuration(false)); tcm.start(); assertEquals(2, tcm.getNumTaskComms()); assertTrue(tcm.getYarnTaskCommCreated()); assertFalse(tcm.getUberTaskCommCreated()); verify(tcm.getTestTaskComm(0)).initialize(); verify(tcm.getTestTaskComm(0)).start(); verify(tcm.getTestTaskComm(1)).initialize(); verify(tcm.getTestTaskComm(1)).start(); ContainerId containerId1 = mock(ContainerId.class); tcm.registerRunningContainer(containerId1, 0); verify(tcm.getTestTaskComm(0)).registerRunningContainer(eq(containerId1), eq("host1"), eq(3131)); ContainerId containerId2 = mock(ContainerId.class); tcm.registerRunningContainer(containerId2, 1); verify(tcm.getTestTaskComm(1)).registerRunningContainer(eq(containerId2), eq("host1"), eq(3131)); } finally { tcm.stop(); verify(tcm.getTaskCommunicator(0).getTaskCommunicator()).shutdown(); verify(tcm.getTaskCommunicator(1).getTaskCommunicator()).shutdown(); } }
From source file:org.apache.tez.dag.app.TestTaskCommunicatorManager1.java
License:Apache License
@Before public void setUp() throws TezException { appId = ApplicationId.newInstance(1000, 1); appAttemptId = ApplicationAttemptId.newInstance(appId, 1); dag = mock(DAG.class); TezDAGID dagID = TezDAGID.getInstance(appId, 1); vertexID = TezVertexID.getInstance(dagID, 1); taskID = TezTaskID.getInstance(vertexID, 1); taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); credentials = new Credentials(); amContainerMap = mock(AMContainerMap.class); Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>(); eventHandler = mock(EventHandler.class); MockClock clock = new MockClock(); appContext = mock(AppContext.class); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(dag).when(appContext).getCurrentDAG(); doReturn(appAcls).when(appContext).getApplicationACLs(); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(clock).when(appContext).getClock(); doReturn(appAttemptId).when(appContext).getApplicationAttemptId(); doReturn(credentials).when(appContext).getAppCredentials(); NodeId nodeId = NodeId.newInstance("localhost", 0); AMContainer amContainer = mock(AMContainer.class); Container container = mock(Container.class); doReturn(nodeId).when(container).getNodeId(); doReturn(amContainer).when(amContainerMap).get(any(ContainerId.class)); doReturn(container).when(amContainer).getContainer(); Configuration conf = new TezConfiguration(); UserPayload defaultPayload;/*from w w w. j a v a 2 s . c om*/ try { defaultPayload = TezUtils.createUserPayloadFromConf(conf); } catch (IOException e) { throw new TezUncheckedException(e); } taskAttemptListener = new TaskCommunicatorManagerInterfaceImplForTest(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), Lists.newArrayList(new NamedEntityDescriptor(TezConstants.getTezYarnServicePluginName(), null) .setUserPayload(defaultPayload))); taskSpec = mock(TaskSpec.class); doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID(); amContainerTask = new AMContainerTask(taskSpec, null, null, false, 0); containerTask = null; }
From source file:org.apache.tez.dag.history.events.TestHistoryEventsProtoConversion.java
License:Apache License
private void testTaskAttemptStartedEvent() throws Exception { TaskAttemptStartedEvent event = new TaskAttemptStartedEvent( TezTaskAttemptID.getInstance(TezTaskID.getInstance( TezVertexID.getInstance(TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 1), 1),//from ww w . jav a 2 s . c o m "vertex1", 10009l, ContainerId.newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1001), NodeId.newInstance("host1", 19999), "inProgress", "Completed", "nodeHttpAddress"); TaskAttemptStartedEvent deserializedEvent = (TaskAttemptStartedEvent) testProtoConversion(event); Assert.assertEquals(event.getTaskAttemptID(), deserializedEvent.getTaskAttemptID()); Assert.assertEquals(event.getContainerId(), deserializedEvent.getContainerId()); Assert.assertEquals(event.getNodeId(), deserializedEvent.getNodeId()); Assert.assertEquals(event.getStartTime(), deserializedEvent.getStartTime()); logEvents(event, deserializedEvent); }