List of usage examples for org.apache.hadoop.yarn.api.records ApplicationId newInstance
@Public
@Unstable
public static ApplicationId newInstance(long clusterTimestamp, int id)
From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl.java
License:Apache License
public void setupPreDagCreation() { LOG.info("____________ RESETTING CURRENT DAG ____________"); conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(100, 1), 1); dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1); taskSpecificLaunchCmdOption = mock(TaskSpecificLaunchCmdOption.class); doReturn(false).when(taskSpecificLaunchCmdOption).addTaskSpecificLaunchCmdOption(any(String.class), anyInt());//w w w . java 2s .co m }
From source file:org.apache.tez.dag.app.dag.impl.TestVertexStats.java
License:Apache License
@Test(timeout = 5000) public void testBasicStats() { VertexStats stats = new VertexStats(); Assert.assertEquals(-1, stats.firstTaskStartTime); Assert.assertEquals(-1, stats.lastTaskFinishTime); Assert.assertEquals(-1, stats.minTaskDuration); Assert.assertEquals(-1, stats.maxTaskDuration); Assert.assertTrue(-1 == stats.avgTaskDuration); Assert.assertEquals(0, stats.getFirstTasksToStart().size()); Assert.assertEquals(0, stats.getLastTasksToFinish().size()); Assert.assertEquals(0, stats.getShortestDurationTasks().size()); Assert.assertEquals(0, stats.getLongestDurationTasks().size()); TezVertexID tezVertexID = TezVertexID .getInstance(TezDAGID.getInstance(ApplicationId.newInstance(100l, 1), 1), 1); TezTaskID tezTaskID1 = TezTaskID.getInstance(tezVertexID, 1); TezTaskID tezTaskID2 = TezTaskID.getInstance(tezVertexID, 2); TezTaskID tezTaskID3 = TezTaskID.getInstance(tezVertexID, 3); TezTaskID tezTaskID4 = TezTaskID.getInstance(tezVertexID, 4); TezTaskID tezTaskID5 = TezTaskID.getInstance(tezVertexID, 5); TezTaskID tezTaskID6 = TezTaskID.getInstance(tezVertexID, 6); stats.updateStats(new TaskReportImpl(tezTaskID1, TaskState.SUCCEEDED, 1, 100, 200)); Assert.assertEquals(100, stats.firstTaskStartTime); Assert.assertEquals(200, stats.lastTaskFinishTime); Assert.assertEquals(100, stats.minTaskDuration); Assert.assertEquals(100, stats.maxTaskDuration); Assert.assertTrue(100 == stats.avgTaskDuration); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID1)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID1)); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID1)); Assert.assertEquals(1, stats.firstTasksToStart.size()); Assert.assertEquals(1, stats.lastTasksToFinish.size()); Assert.assertEquals(1, stats.shortestDurationTasks.size()); Assert.assertEquals(1, stats.longestDurationTasks.size()); stats.updateStats(new TaskReportImpl(tezTaskID2, TaskState.FAILED, 1, 150, 300)); Assert.assertEquals(100, stats.firstTaskStartTime); Assert.assertEquals(300, stats.lastTaskFinishTime); Assert.assertEquals(100, stats.minTaskDuration); Assert.assertEquals(100, stats.maxTaskDuration); Assert.assertTrue(100 == stats.avgTaskDuration); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID1)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID2)); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID1)); Assert.assertEquals(1, stats.firstTasksToStart.size()); Assert.assertEquals(1, stats.lastTasksToFinish.size()); Assert.assertEquals(1, stats.shortestDurationTasks.size()); Assert.assertEquals(1, stats.longestDurationTasks.size()); stats.updateStats(new TaskReportImpl(tezTaskID3, TaskState.RUNNING, 1, 50, 550)); Assert.assertEquals(50, stats.firstTaskStartTime); Assert.assertEquals(550, stats.lastTaskFinishTime); Assert.assertEquals(100, stats.minTaskDuration); Assert.assertEquals(100, stats.maxTaskDuration); Assert.assertTrue(100 == stats.avgTaskDuration); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3)); Assert.assertEquals(1, stats.firstTasksToStart.size()); Assert.assertEquals(1, stats.lastTasksToFinish.size()); Assert.assertEquals(1, stats.shortestDurationTasks.size()); Assert.assertEquals(1, stats.longestDurationTasks.size()); stats.updateStats(new TaskReportImpl(tezTaskID4, TaskState.SUCCEEDED, 1, 50, 450)); Assert.assertEquals(50, stats.firstTaskStartTime); Assert.assertEquals(550, stats.lastTaskFinishTime); Assert.assertEquals(100, stats.minTaskDuration); Assert.assertEquals(400, stats.maxTaskDuration); Assert.assertTrue(250 == stats.avgTaskDuration); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID4)); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3)); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID4)); Assert.assertEquals(2, stats.firstTasksToStart.size()); Assert.assertEquals(1, stats.lastTasksToFinish.size()); Assert.assertEquals(1, stats.shortestDurationTasks.size()); Assert.assertEquals(1, stats.longestDurationTasks.size()); stats.updateStats(new TaskReportImpl(tezTaskID5, TaskState.SUCCEEDED, 1, 50, 450)); Assert.assertEquals(50, stats.firstTaskStartTime); Assert.assertEquals(550, stats.lastTaskFinishTime); Assert.assertEquals(100, stats.minTaskDuration); Assert.assertEquals(400, stats.maxTaskDuration); Assert.assertTrue(300 == stats.avgTaskDuration); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID5)); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID4)); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3)); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID4)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID5)); Assert.assertEquals(3, stats.firstTasksToStart.size()); Assert.assertEquals(1, stats.lastTasksToFinish.size()); Assert.assertEquals(1, stats.shortestDurationTasks.size()); Assert.assertEquals(2, stats.longestDurationTasks.size()); stats.updateStats(new TaskReportImpl(tezTaskID6, TaskState.SUCCEEDED, 1, 450, 550)); Assert.assertEquals(50, stats.firstTaskStartTime); Assert.assertEquals(550, stats.lastTaskFinishTime); Assert.assertEquals(100, stats.minTaskDuration); Assert.assertEquals(400, stats.maxTaskDuration); Assert.assertTrue(250 == stats.avgTaskDuration); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID5)); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID4)); Assert.assertTrue(stats.firstTasksToStart.contains(tezTaskID3)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID3)); Assert.assertTrue(stats.lastTasksToFinish.contains(tezTaskID6)); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID1)); Assert.assertTrue(stats.shortestDurationTasks.contains(tezTaskID6)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID4)); Assert.assertTrue(stats.longestDurationTasks.contains(tezTaskID5)); Assert.assertEquals(3, stats.firstTasksToStart.size()); Assert.assertEquals(2, stats.lastTasksToFinish.size()); Assert.assertEquals(2, stats.shortestDurationTasks.size()); Assert.assertEquals(2, stats.longestDurationTasks.size()); }
From source file:org.apache.tez.dag.app.dag.TestRootInputInitializerManager.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from w w w.j a va 2 s. c o m*/ public void testEventBeforeSuccess() throws Exception { InputDescriptor id = mock(InputDescriptor.class); InputInitializerDescriptor iid = mock(InputInitializerDescriptor.class); RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>( "InputName", id, iid); InputInitializer initializer = mock(InputInitializer.class); InputInitializerContext initializerContext = mock(InputInitializerContext.class); Vertex vertex = mock(Vertex.class); StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class); AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS); RootInputInitializerManager.InitializerWrapper initializerWrapper = new RootInputInitializerManager.InitializerWrapper( rootInput, initializer, initializerContext, vertex, stateChangeNotifier, appContext); ApplicationId appId = ApplicationId.newInstance(1000, 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID srcVertexId = TezVertexID.getInstance(dagId, 2); TezTaskID srcTaskId1 = TezTaskID.getInstance(srcVertexId, 3); Vertex srcVertex = mock(Vertex.class); Task srcTask1 = mock(Task.class); doReturn(TaskState.RUNNING).when(srcTask1).getState(); doReturn(srcTask1).when(srcVertex).getTask(srcTaskId1.getId()); when(appContext.getCurrentDAG().getVertex(any(String.class))).thenReturn(srcVertex); String srcVertexName = "srcVertexName"; List<TezEvent> eventList = Lists.newLinkedList(); // First Attempt send event TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1); EventMetaData sourceInfo11 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId11); InputInitializerEvent e1 = InputInitializerEvent.create("fakeVertex", "fakeInput", null); TezEvent te1 = new TezEvent(e1, sourceInfo11); eventList.add(te1); initializerWrapper.handleInputInitializerEvents(eventList); verify(initializer, never()).handleInputInitializerEvent(any(List.class)); eventList.clear(); // First attempt, Task success notification initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId11.getId()); ArgumentCaptor<List> argumentCaptor = ArgumentCaptor.forClass(List.class); verify(initializer, times(1)).handleInputInitializerEvent(argumentCaptor.capture()); List<InputInitializerEvent> invokedEvents = argumentCaptor.getValue(); assertEquals(1, invokedEvents.size()); reset(initializer); // 2nd attempt send event TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2); EventMetaData sourceInfo12 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId12); InputInitializerEvent e2 = InputInitializerEvent.create("fakeVertex", "fakeInput", null); TezEvent te2 = new TezEvent(e2, sourceInfo12); eventList.add(te2); initializerWrapper.handleInputInitializerEvents(eventList); verify(initializer, never()).handleInputInitializerEvent(any(List.class)); eventList.clear(); reset(initializer); // 2nd attempt succeeded initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId()); verify(initializer, never()).handleInputInitializerEvent(argumentCaptor.capture()); }
From source file:org.apache.tez.dag.app.dag.TestRootInputInitializerManager.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)// www. j a v a2 s . c o m public void testSuccessBeforeEvent() throws Exception { InputDescriptor id = mock(InputDescriptor.class); InputInitializerDescriptor iid = mock(InputInitializerDescriptor.class); RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput = new RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>( "InputName", id, iid); InputInitializer initializer = mock(InputInitializer.class); InputInitializerContext initializerContext = mock(InputInitializerContext.class); Vertex vertex = mock(Vertex.class); StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class); AppContext appContext = mock(AppContext.class, RETURNS_DEEP_STUBS); RootInputInitializerManager.InitializerWrapper initializerWrapper = new RootInputInitializerManager.InitializerWrapper( rootInput, initializer, initializerContext, vertex, stateChangeNotifier, appContext); ApplicationId appId = ApplicationId.newInstance(1000, 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID srcVertexId = TezVertexID.getInstance(dagId, 2); TezTaskID srcTaskId1 = TezTaskID.getInstance(srcVertexId, 3); Vertex srcVertex = mock(Vertex.class); Task srcTask1 = mock(Task.class); doReturn(TaskState.RUNNING).when(srcTask1).getState(); doReturn(srcTask1).when(srcVertex).getTask(srcTaskId1.getId()); when(appContext.getCurrentDAG().getVertex(any(String.class))).thenReturn(srcVertex); String srcVertexName = "srcVertexName"; List<TezEvent> eventList = Lists.newLinkedList(); // First Attempt send event TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1); EventMetaData sourceInfo11 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId11); InputInitializerEvent e1 = InputInitializerEvent.create("fakeVertex", "fakeInput", null); TezEvent te1 = new TezEvent(e1, sourceInfo11); eventList.add(te1); initializerWrapper.handleInputInitializerEvents(eventList); verify(initializer, never()).handleInputInitializerEvent(any(List.class)); eventList.clear(); // First attempt, Task success notification initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId11.getId()); ArgumentCaptor<List> argumentCaptor = ArgumentCaptor.forClass(List.class); verify(initializer, times(1)).handleInputInitializerEvent(argumentCaptor.capture()); List<InputInitializerEvent> invokedEvents = argumentCaptor.getValue(); assertEquals(1, invokedEvents.size()); reset(initializer); TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2); // 2nd attempt succeeded initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId()); verify(initializer, never()).handleInputInitializerEvent(any(List.class)); // 2nd attempt send event EventMetaData sourceInfo12 = new EventMetaData(EventMetaData.EventProducerConsumerType.PROCESSOR, srcVertexName, null, srcTaskAttemptId12); InputInitializerEvent e2 = InputInitializerEvent.create("fakeVertex", "fakeInput", null); TezEvent te2 = new TezEvent(e2, sourceInfo12); eventList.add(te2); initializerWrapper.handleInputInitializerEvents(eventList); verify(initializer, never()).handleInputInitializerEvent(any(List.class)); }
From source file:org.apache.tez.dag.app.launcher.TestContainerLauncherManager.java
License:Apache License
@SuppressWarnings("unchecked") @Test(timeout = 5000)/*from w w w. j a v a 2s .c o m*/ public void testReportFailureFromContainerLauncher() throws ServicePluginException, TezException { final String dagName = DAG_NAME; final int dagIndex = DAG_INDEX; TezDAGID dagId = TezDAGID.getInstance(ApplicationId.newInstance(0, 0), dagIndex); DAG dag = mock(DAG.class); doReturn(dagName).when(dag).getName(); doReturn(dagId).when(dag).getID(); EventHandler eventHandler = mock(EventHandler.class); AppContext appContext = mock(AppContext.class); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(dag).when(appContext).getCurrentDAG(); doReturn("testlauncher").when(appContext).getContainerLauncherName(0); NamedEntityDescriptor<TaskCommunicatorDescriptor> taskCommDescriptor = new NamedEntityDescriptor<>( "testlauncher", ContainerLauncherForTest.class.getName()); List<NamedEntityDescriptor> list = new LinkedList<>(); list.add(taskCommDescriptor); ContainerLauncherManager containerLauncherManager = new ContainerLauncherManager(appContext, mock(TaskCommunicatorManagerInterface.class), "", list, false); try { ContainerLaunchContext clc1 = mock(ContainerLaunchContext.class); Container container1 = mock(Container.class); ContainerLauncherLaunchRequestEvent launchRequestEvent = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0); containerLauncherManager.handle(launchRequestEvent); ArgumentCaptor<Event> argumentCaptor = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(1)).handle(argumentCaptor.capture()); Event rawEvent = argumentCaptor.getValue(); assertTrue(rawEvent instanceof DAGAppMasterEventUserServiceFatalError); DAGAppMasterEventUserServiceFatalError event = (DAGAppMasterEventUserServiceFatalError) rawEvent; assertEquals(DAGAppMasterEventType.CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR, event.getType()); assertTrue(event.getDiagnosticInfo().contains("ReportedFatalError")); assertTrue(event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name())); assertTrue(event.getDiagnosticInfo().contains("[0:testlauncher]")); reset(eventHandler); // stop container ContainerId containerId2 = mock(ContainerId.class); NodeId nodeId2 = mock(NodeId.class); ContainerLauncherStopRequestEvent stopRequestEvent = new ContainerLauncherStopRequestEvent(containerId2, nodeId2, null, 0, 0, 0); argumentCaptor = ArgumentCaptor.forClass(Event.class); containerLauncherManager.handle(stopRequestEvent); verify(eventHandler, times(1)).handle(argumentCaptor.capture()); rawEvent = argumentCaptor.getValue(); assertTrue(rawEvent instanceof DAGEventTerminateDag); DAGEventTerminateDag killEvent = (DAGEventTerminateDag) rawEvent; assertTrue(killEvent.getDiagnosticInfo().contains("ReportError")); assertTrue( killEvent.getDiagnosticInfo().contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())); assertTrue(killEvent.getDiagnosticInfo().contains("[0:testlauncher]")); } finally { containerLauncherManager.stop(); } }
From source file:org.apache.tez.dag.app.rm.container.TestAMContainerMap.java
License:Apache License
private ContainerId mockContainerId(int cId) { ApplicationId appId = ApplicationId.newInstance(1000, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId.newInstance(appAttemptId, cId); return containerId; }
From source file:org.apache.tez.dag.app.rm.ContainerFactory.java
License:Apache License
public ContainerFactory(ApplicationAttemptId appAttemptId, long appIdLong) { this.nextId = new AtomicLong(1); ApplicationId appId = ApplicationId.newInstance(appIdLong, appAttemptId.getApplicationId().getId()); this.customAppAttemptId = ApplicationAttemptId.newInstance(appId, appAttemptId.getAttemptId()); }
From source file:org.apache.tez.dag.app.rm.TestContainerReuse.java
License:Apache License
private Container createContainer(int id, String host, Resource resource, Priority priority) { ContainerId containerID = ContainerId .newInstance(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), id); NodeId nodeID = NodeId.newInstance(host, 0); Container container = Container.newInstance(containerID, nodeID, host + ":0", resource, priority, null); return container; }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked" })
@Test(timeout = 30000)//from ww w . jav a2s. c om
public void testNoReuse() throws Exception {
AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest());
String appHost = "host";
int appPort = 0;
String appUrl = "url";
Configuration conf = new Configuration();
conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100);
DagInfo mockDagInfo = mock(DagInfo.class);
when(mockDagInfo.getTotalVertices()).thenReturn(10);
when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet());
TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf);
when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo);
TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp);
MockClock clock = new MockClock(1000);
NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock);
scheduler.initialize();
drainableAppCallback.drain();
scheduler.start();
drainableAppCallback.drain();
verify(mockRMClient).start();
verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl);
RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse();
verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(),
regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(),
regResponse.getQueue());
assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount());
Object mockTask1 = new MockTask("task1");
Object mockCookie1 = new Object();
Resource mockCapability = Resources.createResource(1024, 1);
String[] hosts = { "host1", "host5" };
String[] racks = { "/default-rack", "/default-rack" };
Priority mockPriority = Priority.newInstance(1);
ArgumentCaptor<TaskRequest> requestCaptor = ArgumentCaptor.forClass(TaskRequest.class);
// allocate task
scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1);
drainableAppCallback.drain();
verify(mockRMClient, times(1)).addContainerRequest(any(TaskRequest.class));
// returned from task requests before allocation happens
assertFalse(scheduler.deallocateTask(mockTask1, true, null, null));
verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class));
verify(mockRMClient, times(1)).removeContainerRequest(any(TaskRequest.class));
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// deallocating unknown task
assertFalse(scheduler.deallocateTask(mockTask1, true, null, null));
verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class));
verify(mockRMClient, times(1)).removeContainerRequest(any(TaskRequest.class));
verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any());
// allocate tasks
Object mockTask2 = new MockTask("task2");
Object mockCookie2 = new Object();
Object mockTask3 = new MockTask("task3");
Object mockCookie3 = new Object();
scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1);
drainableAppCallback.drain();
verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture());
TaskRequest request1 = requestCaptor.getValue();
scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority, null, mockCookie2);
drainableAppCallback.drain();
verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture());
TaskRequest request2 = requestCaptor.getValue();
scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority, null, mockCookie3);
drainableAppCallback.drain();
verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture());
TaskRequest request3 = requestCaptor.getValue();
NodeId host1 = NodeId.newInstance("host1", 1);
NodeId host2 = NodeId.newInstance("host2", 2);
NodeId host3 = NodeId.newInstance("host3", 3);
NodeId host4 = NodeId.newInstance("host4", 4);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
ContainerId mockCId1 = ContainerId.newContainerId(attemptId, 1);
Container mockContainer1 = Container.newInstance(mockCId1, host1, null, mockCapability, mockPriority, null);
ContainerId mockCId2 = ContainerId.newContainerId(attemptId, 2);
Container mockContainer2 = Container.newInstance(mockCId2, host2, null, mockCapability, mockPriority, null);
ContainerId mockCId3 = ContainerId.newContainerId(attemptId, 3);
Container mockContainer3 = Container.newInstance(mockCId3, host3, null, mockCapability, mockPriority, null);
ContainerId mockCId4 = ContainerId.newContainerId(attemptId, 4);
Container mockContainer4 = Container.newInstance(mockCId4, host4, null, mockCapability, mockPriority, null);
List<Container> containers = new ArrayList<>();
containers.add(mockContainer1);
containers.add(mockContainer2);
containers.add(mockContainer3);
containers.add(mockContainer4);
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
// first container allocated
verify(mockApp).taskAllocated(mockTask1, mockCookie1, mockContainer1);
verify(mockApp).taskAllocated(mockTask2, mockCookie2, mockContainer2);
verify(mockApp).taskAllocated(mockTask3, mockCookie3, mockContainer3);
// no other allocations returned
verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any());
verify(mockRMClient).removeContainerRequest(request1);
verify(mockRMClient).removeContainerRequest(request2);
verify(mockRMClient).removeContainerRequest(request3);
// verify unwanted container released
verify(mockRMClient).releaseAssignedContainer(mockCId4);
// deallocate allocated task
assertTrue(scheduler.deallocateTask(mockTask1, true, null, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId1);
verify(mockRMClient).releaseAssignedContainer(mockCId1);
// deallocate allocated container
assertEquals(mockTask2, scheduler.deallocateContainer(mockCId2));
drainableAppCallback.drain();
verify(mockRMClient).releaseAssignedContainer(mockCId2);
verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any());
List<ContainerStatus> statuses = new ArrayList<>();
ContainerStatus mockStatus1 = mock(ContainerStatus.class);
when(mockStatus1.getContainerId()).thenReturn(mockCId1);
statuses.add(mockStatus1);
ContainerStatus mockStatus2 = mock(ContainerStatus.class);
when(mockStatus2.getContainerId()).thenReturn(mockCId2);
statuses.add(mockStatus2);
ContainerStatus mockStatus3 = mock(ContainerStatus.class);
when(mockStatus3.getContainerId()).thenReturn(mockCId3);
statuses.add(mockStatus3);
ContainerStatus mockStatus4 = mock(ContainerStatus.class);
when(mockStatus4.getContainerId()).thenReturn(mockCId4);
statuses.add(mockStatus4);
scheduler.onContainersCompleted(statuses);
drainableAppCallback.drain();
// released container status returned
verify(mockApp).containerCompleted(mockTask1, mockStatus1);
verify(mockApp).containerCompleted(mockTask2, mockStatus2);
// currently allocated container status returned and not released
verify(mockApp).containerCompleted(mockTask3, mockStatus3);
// no other statuses returned
verify(mockApp, times(3)).containerCompleted(any(), (ContainerStatus) any());
verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any());
// verify blacklisting
verify(mockRMClient, times(0)).updateBlacklist(anyListOf(String.class), anyListOf(String.class));
String badHost = "host6";
NodeId badNodeId = NodeId.newInstance(badHost, 1);
scheduler.blacklistNode(badNodeId);
List<String> badNodeList = Collections.singletonList(badHost);
verify(mockRMClient, times(1)).updateBlacklist(eq(badNodeList), isNull(List.class));
Object mockTask4 = new MockTask("task4");
Object mockCookie4 = new Object();
scheduler.allocateTask(mockTask4, mockCapability, null, null, mockPriority, null, mockCookie4);
drainableAppCallback.drain();
verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture());
ContainerId mockCId5 = ContainerId.newContainerId(attemptId, 5);
Container mockContainer5 = Container.newInstance(mockCId5, badNodeId, null, mockCapability, mockPriority,
null);
containers.clear();
containers.add(mockContainer5);
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
// no new allocation
verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any());
// verify blacklisted container released
verify(mockRMClient).releaseAssignedContainer(mockCId5);
verify(mockRMClient, times(4)).releaseAssignedContainer((ContainerId) any());
// verify request added back
verify(mockRMClient, times(6)).addContainerRequest(requestCaptor.capture());
NodeId host6 = NodeId.newInstance("host6", 6);
ContainerId mockCId6 = ContainerId.newContainerId(attemptId, 6);
Container mockContainer6 = Container.newInstance(mockCId6, host6, null, mockCapability, mockPriority, null);
containers.clear();
containers.add(mockContainer6);
scheduler.onContainersAllocated(containers);
drainableAppCallback.drain();
// new allocation
verify(mockApp, times(4)).taskAllocated(any(), any(), (Container) any());
verify(mockApp).taskAllocated(mockTask4, mockCookie4, mockContainer6);
// deallocate allocated task
assertTrue(scheduler.deallocateTask(mockTask4, true, null, null));
drainableAppCallback.drain();
verify(mockApp).containerBeingReleased(mockCId6);
verify(mockRMClient).releaseAssignedContainer(mockCId6);
verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any());
// test unblacklist
scheduler.unblacklistNode(badNodeId);
verify(mockRMClient, times(1)).updateBlacklist(isNull(List.class), eq(badNodeList));
assertEquals(0, scheduler.getNumBlacklistedNodes());
float progress = 0.5f;
when(mockApp.getProgress()).thenReturn(progress);
assertEquals(progress, scheduler.getProgress(), 0);
// check duplicate allocation request
scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1);
drainableAppCallback.drain();
verify(mockRMClient, times(7)).addContainerRequest(any(TaskRequest.class));
verify(mockRMClient, times(6)).removeContainerRequest(any(TaskRequest.class));
scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1);
drainableAppCallback.drain();
// old request removed and new one added
verify(mockRMClient, times(7)).removeContainerRequest(any(TaskRequest.class));
verify(mockRMClient, times(8)).addContainerRequest(any(TaskRequest.class));
assertFalse(scheduler.deallocateTask(mockTask1, true, null, null));
List<NodeReport> mockUpdatedNodes = mock(List.class);
scheduler.onNodesUpdated(mockUpdatedNodes);
drainableAppCallback.drain();
verify(mockApp).nodesUpdated(mockUpdatedNodes);
ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
Exception mockException = new IOException("mockexception");
scheduler.onError(mockException);
drainableAppCallback.drain();
verify(mockApp).reportError(eq(YarnTaskSchedulerServiceError.RESOURCEMANAGER_ERROR),
argumentCaptor.capture(), any(DagInfo.class));
assertTrue(argumentCaptor.getValue().contains("mockexception"));
scheduler.onShutdownRequest();
drainableAppCallback.drain();
verify(mockApp).appShutdownRequested();
String appMsg = "success";
AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl);
when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
scheduler.shutdown();
drainableAppCallback.drain();
verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl);
verify(mockRMClient).stop();
}
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testSimpleReuseLocalMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();// w w w. ja v a2s . c o m drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host3"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, hostsv0t0); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, hostsv0t0); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, hostsv0t0); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); taskRequestCaptor.scheduleTask(taskv0t1); TaskRequest reqv0t2 = taskRequestCaptor.scheduleTask(taskv0t2); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // finish v0t0 successfully, verify v0t1 is skipped and v0t2 instead is assigned to the container assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t2.task, taskv0t2.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t2); // finish v0t2 successfully, verify v1t0 is assigned to the same container assertTrue(scheduler.deallocateTask(taskv0t2.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // fail v1t0 and verify container is released instead of reused for v1t1 assertTrue(scheduler.deallocateTask(taskv1t0.task, false, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }