List of usage examples for org.apache.hadoop.yarn.api.records ContainerId newContainerId
@Public @Unstable public static ContainerId newContainerId(ApplicationAttemptId appAttemptId, long containerId)
From source file:UnmanagedAMLauncher.java
License:Apache License
public void launchAM(ApplicationAttemptId attemptId) throws IOException, YarnException { Credentials credentials = new Credentials(); Token<AMRMTokenIdentifier> token = rmClient.getAMRMToken(attemptId.getApplicationId()); // Service will be empty but that's okay, we are just passing down only // AMRMToken down to the real AM which eventually sets the correct // service-address. credentials.addToken(token.getService(), token); File tokenFile = File.createTempFile("unmanagedAMRMToken", "", new File(System.getProperty("user.dir"))); try {/*from ww w.ja v a 2s . c om*/ FileUtil.chmod(tokenFile.getAbsolutePath(), "600"); } catch (InterruptedException ex) { throw new RuntimeException(ex); } tokenFile.deleteOnExit(); DataOutputStream os = new DataOutputStream(new FileOutputStream(tokenFile, true)); credentials.writeTokenStorageToStream(os); os.close(); Map<String, String> env = System.getenv(); ArrayList<String> envAMList = new ArrayList<String>(); boolean setClasspath = false; for (Map.Entry<String, String> entry : env.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (key.equals("CLASSPATH")) { setClasspath = true; if (classpath != null) { value = value + File.pathSeparator + classpath; } } envAMList.add(key + "=" + value); } if (!setClasspath && classpath != null) { envAMList.add("CLASSPATH=" + classpath); } ContainerId containerId = ContainerId.newContainerId(attemptId, 0); String hostname = InetAddress.getLocalHost().getHostName(); envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId); envAMList.add(Environment.NM_HOST.name() + "=" + hostname); envAMList.add(Environment.NM_HTTP_PORT.name() + "=0"); envAMList.add(Environment.NM_PORT.name() + "=0"); envAMList.add(Environment.LOCAL_DIRS.name() + "= /tmp"); envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + System.currentTimeMillis()); envAMList.add(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME + "=" + tokenFile.getAbsolutePath()); String[] envAM = new String[envAMList.size()]; Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM)); final BufferedReader errReader = new BufferedReader(new InputStreamReader(amProc.getErrorStream())); final BufferedReader inReader = new BufferedReader(new InputStreamReader(amProc.getInputStream())); // read error and input streams as this would free up the buffers // free the error stream buffer Thread errThread = new Thread() { @Override public void run() { try { String line = errReader.readLine(); while ((line != null) && !isInterrupted()) { System.err.println(line); line = errReader.readLine(); } } catch (IOException ioe) { LOG.warn("Error reading the error stream", ioe); } } }; Thread outThread = new Thread() { @Override public void run() { try { String line = inReader.readLine(); while ((line != null) && !isInterrupted()) { System.out.println(line); line = inReader.readLine(); } } catch (IOException ioe) { LOG.warn("Error reading the out stream", ioe); } } }; try { errThread.start(); outThread.start(); } catch (IllegalStateException ise) { } // wait for the process to finish and check the exit code try { int exitCode = amProc.waitFor(); LOG.info("AM process exited with value: " + exitCode); } catch (InterruptedException e) { e.printStackTrace(); } finally { amCompleted = true; } try { // make sure that the error thread exits // on Windows these threads sometimes get stuck and hang the execution // timeout and join later after destroying the process. errThread.join(); outThread.join(); errReader.close(); inReader.close(); } catch (InterruptedException ie) { LOG.info("ShellExecutor: Interrupted while reading the error/out stream", ie); } catch (IOException ioe) { LOG.warn("Error while closing the error/out stream", ioe); } amProc.destroy(); }
From source file:de.huberlin.wbi.hiway.scheduler.TestScheduler.java
License:Apache License
private static void run(Scheduler scheduler, List<String> nodeNames, List<String> taskNames, int[][] runtimes) { Queue<NodeId> availableNodes = new LinkedList<>(); for (String nodeName : nodeNames) { NodeId node = NodeId.newInstance(nodeName, 0); availableNodes.add(node);/*from ww w .ja v a2s . c om*/ } Map<Container, TaskInstance> runningTasks = new HashMap<>(); Map<Container, Long> finishTimes = new HashMap<>(); int runningId = 0; long clock = 0; while (!scheduler.nothingToSchedule() || !runningTasks.isEmpty()) { if (!scheduler.nothingToSchedule() && !availableNodes.isEmpty()) { NodeId nodeId = availableNodes.remove(); ContainerId containerId = ContainerId.newContainerId(null, runningId++); Container container = Container.newInstance(containerId, nodeId, "", null, null, null); TaskInstance task = scheduler.getNextTask(container); runningTasks.put(container, task); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; finishTimes.put(container, clock + runtime); } for (Container container : finishTimes.keySet()) { if (clock == finishTimes.get(container)) { NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = ContainerStatus.newInstance(container.getId(), null, "", 0); TaskInstance task = runningTasks.get(container); task.setCompleted(); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; scheduler.taskCompleted(task, containerStatus, runtime); runningTasks.remove(container); availableNodes.add(nodeId); } } clock++; } }
From source file:org.apache.ignite.yarn.IgniteApplicationMasterSelfTest.java
License:Apache License
/** * @param host Host.//w ww . ja v a 2 s.c o m * @param cpu Cpu count. * @param mem Memory. * @return Container. */ private Container createContainer(String host, int cpu, int mem) { return Container.newInstance( ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0), ThreadLocalRandom.current().nextLong()), NodeId.newInstance(host, 0), "example.com", new MockResource(mem, cpu), Priority.newInstance(0), null); }
From source file:org.apache.myriad.TestObjectFactory.java
License:Apache License
/** * Returns a new RMContainer corresponding to the RMNode and RMContext. The RMContainer is the * ResourceManager's view of an application container per the Hadoop docs * //from w w w . ja v a 2 s. c o m * @param node * @param context * @param appId * @param cores * @param memory * @return RMContainer */ public static RMContainer getRMContainer(RMNode node, RMContext context, int appId, int cores, int memory) { ContainerId containerId = ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(123456789, 1), 1), appId); Container container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(), Resources.createResource(memory, cores), null, null); return new RMContainerImpl(container, containerId.getApplicationAttemptId(), node.getNodeID(), "user1", context); }
From source file:org.apache.samza.job.yarn.TestYarnClusterResourceManager.java
License:Apache License
@Test public void testErrorInStartContainerShouldUpdateState() { // create mocks final int samzaContainerId = 1; YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class); SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class); Config config = mock(Config.class); AMRMClientAsync asyncClient = mock(AMRMClientAsync.class); YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081); SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class); SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class); NMClientAsync asyncNMClient = mock(NMClientAsync.class); ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class); // start the cluster manager YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config); yarnAppState.pendingProcessors.put(String.valueOf(samzaContainerId), new YarnContainer(Container.newInstance( ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1), 1), NodeId.newInstance("host1", 8088), "http://host1", Resource.newInstance(1024, 1), Priority.newInstance(1), Token.newInstance("id".getBytes(), "read", "password".getBytes(), "service")))); yarnClusterResourceManager.start();/*from w ww.j a v a 2s.c o m*/ assertEquals(1, yarnAppState.pendingProcessors.size()); yarnClusterResourceManager.onStartContainerError( ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1), 1), new Exception()); assertEquals(0, yarnAppState.pendingProcessors.size()); verify(callback, times(1)).onStreamProcessorLaunchFailure(anyObject(), any(Exception.class)); }
From source file:org.apache.tez.dag.app.rm.ContainerFactory.java
License:Apache License
public Container createContainer(Resource capability, Priority priority, String hostname, int port) { ContainerId containerId = ContainerId.newContainerId(customAppAttemptId, nextId.getAndIncrement()); NodeId nodeId = NodeId.newInstance(hostname, port); String nodeHttpAddress = "hostname:0"; // TODO: include UI ports Container container = Container.newInstance(containerId, nodeId, nodeHttpAddress, capability, priority, null);/*from ww w . ja v a2 s . com*/ return container; }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@SuppressWarnings({ "unchecked" }) @Test(timeout = 30000)//from w ww .j a v a 2 s . com public void testNoReuse() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize(); drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Object mockTask1 = new MockTask("task1"); Object mockCookie1 = new Object(); Resource mockCapability = Resources.createResource(1024, 1); String[] hosts = { "host1", "host5" }; String[] racks = { "/default-rack", "/default-rack" }; Priority mockPriority = Priority.newInstance(1); ArgumentCaptor<TaskRequest> requestCaptor = ArgumentCaptor.forClass(TaskRequest.class); // allocate task scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(1)).addContainerRequest(any(TaskRequest.class)); // returned from task requests before allocation happens assertFalse(scheduler.deallocateTask(mockTask1, true, null, null)); verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, times(1)).removeContainerRequest(any(TaskRequest.class)); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // deallocating unknown task assertFalse(scheduler.deallocateTask(mockTask1, true, null, null)); verify(mockApp, times(0)).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, times(1)).removeContainerRequest(any(TaskRequest.class)); verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // allocate tasks Object mockTask2 = new MockTask("task2"); Object mockCookie2 = new Object(); Object mockTask3 = new MockTask("task3"); Object mockCookie3 = new Object(); scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(2)).addContainerRequest(requestCaptor.capture()); TaskRequest request1 = requestCaptor.getValue(); scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(3)).addContainerRequest(requestCaptor.capture()); TaskRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority, null, mockCookie3); drainableAppCallback.drain(); verify(mockRMClient, times(4)).addContainerRequest(requestCaptor.capture()); TaskRequest request3 = requestCaptor.getValue(); NodeId host1 = NodeId.newInstance("host1", 1); NodeId host2 = NodeId.newInstance("host2", 2); NodeId host3 = NodeId.newInstance("host3", 3); NodeId host4 = NodeId.newInstance("host4", 4); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId mockCId1 = ContainerId.newContainerId(attemptId, 1); Container mockContainer1 = Container.newInstance(mockCId1, host1, null, mockCapability, mockPriority, null); ContainerId mockCId2 = ContainerId.newContainerId(attemptId, 2); Container mockContainer2 = Container.newInstance(mockCId2, host2, null, mockCapability, mockPriority, null); ContainerId mockCId3 = ContainerId.newContainerId(attemptId, 3); Container mockContainer3 = Container.newInstance(mockCId3, host3, null, mockCapability, mockPriority, null); ContainerId mockCId4 = ContainerId.newContainerId(attemptId, 4); Container mockContainer4 = Container.newInstance(mockCId4, host4, null, mockCapability, mockPriority, null); List<Container> containers = new ArrayList<>(); containers.add(mockContainer1); containers.add(mockContainer2); containers.add(mockContainer3); containers.add(mockContainer4); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); // first container allocated verify(mockApp).taskAllocated(mockTask1, mockCookie1, mockContainer1); verify(mockApp).taskAllocated(mockTask2, mockCookie2, mockContainer2); verify(mockApp).taskAllocated(mockTask3, mockCookie3, mockContainer3); // no other allocations returned verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any()); verify(mockRMClient).removeContainerRequest(request1); verify(mockRMClient).removeContainerRequest(request2); verify(mockRMClient).removeContainerRequest(request3); // verify unwanted container released verify(mockRMClient).releaseAssignedContainer(mockCId4); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask1, true, null, null)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId1); verify(mockRMClient).releaseAssignedContainer(mockCId1); // deallocate allocated container assertEquals(mockTask2, scheduler.deallocateContainer(mockCId2)); drainableAppCallback.drain(); verify(mockRMClient).releaseAssignedContainer(mockCId2); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); List<ContainerStatus> statuses = new ArrayList<>(); ContainerStatus mockStatus1 = mock(ContainerStatus.class); when(mockStatus1.getContainerId()).thenReturn(mockCId1); statuses.add(mockStatus1); ContainerStatus mockStatus2 = mock(ContainerStatus.class); when(mockStatus2.getContainerId()).thenReturn(mockCId2); statuses.add(mockStatus2); ContainerStatus mockStatus3 = mock(ContainerStatus.class); when(mockStatus3.getContainerId()).thenReturn(mockCId3); statuses.add(mockStatus3); ContainerStatus mockStatus4 = mock(ContainerStatus.class); when(mockStatus4.getContainerId()).thenReturn(mockCId4); statuses.add(mockStatus4); scheduler.onContainersCompleted(statuses); drainableAppCallback.drain(); // released container status returned verify(mockApp).containerCompleted(mockTask1, mockStatus1); verify(mockApp).containerCompleted(mockTask2, mockStatus2); // currently allocated container status returned and not released verify(mockApp).containerCompleted(mockTask3, mockStatus3); // no other statuses returned verify(mockApp, times(3)).containerCompleted(any(), (ContainerStatus) any()); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); // verify blacklisting verify(mockRMClient, times(0)).updateBlacklist(anyListOf(String.class), anyListOf(String.class)); String badHost = "host6"; NodeId badNodeId = NodeId.newInstance(badHost, 1); scheduler.blacklistNode(badNodeId); List<String> badNodeList = Collections.singletonList(badHost); verify(mockRMClient, times(1)).updateBlacklist(eq(badNodeList), isNull(List.class)); Object mockTask4 = new MockTask("task4"); Object mockCookie4 = new Object(); scheduler.allocateTask(mockTask4, mockCapability, null, null, mockPriority, null, mockCookie4); drainableAppCallback.drain(); verify(mockRMClient, times(5)).addContainerRequest(requestCaptor.capture()); ContainerId mockCId5 = ContainerId.newContainerId(attemptId, 5); Container mockContainer5 = Container.newInstance(mockCId5, badNodeId, null, mockCapability, mockPriority, null); containers.clear(); containers.add(mockContainer5); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); // no new allocation verify(mockApp, times(3)).taskAllocated(any(), any(), (Container) any()); // verify blacklisted container released verify(mockRMClient).releaseAssignedContainer(mockCId5); verify(mockRMClient, times(4)).releaseAssignedContainer((ContainerId) any()); // verify request added back verify(mockRMClient, times(6)).addContainerRequest(requestCaptor.capture()); NodeId host6 = NodeId.newInstance("host6", 6); ContainerId mockCId6 = ContainerId.newContainerId(attemptId, 6); Container mockContainer6 = Container.newInstance(mockCId6, host6, null, mockCapability, mockPriority, null); containers.clear(); containers.add(mockContainer6); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); // new allocation verify(mockApp, times(4)).taskAllocated(any(), any(), (Container) any()); verify(mockApp).taskAllocated(mockTask4, mockCookie4, mockContainer6); // deallocate allocated task assertTrue(scheduler.deallocateTask(mockTask4, true, null, null)); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(mockCId6); verify(mockRMClient).releaseAssignedContainer(mockCId6); verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); // test unblacklist scheduler.unblacklistNode(badNodeId); verify(mockRMClient, times(1)).updateBlacklist(isNull(List.class), eq(badNodeList)); assertEquals(0, scheduler.getNumBlacklistedNodes()); float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); assertEquals(progress, scheduler.getProgress(), 0); // check duplicate allocation request scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(7)).addContainerRequest(any(TaskRequest.class)); verify(mockRMClient, times(6)).removeContainerRequest(any(TaskRequest.class)); scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); // old request removed and new one added verify(mockRMClient, times(7)).removeContainerRequest(any(TaskRequest.class)); verify(mockRMClient, times(8)).addContainerRequest(any(TaskRequest.class)); assertFalse(scheduler.deallocateTask(mockTask1, true, null, null)); List<NodeReport> mockUpdatedNodes = mock(List.class); scheduler.onNodesUpdated(mockUpdatedNodes); drainableAppCallback.drain(); verify(mockApp).nodesUpdated(mockUpdatedNodes); ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class); Exception mockException = new IOException("mockexception"); scheduler.onError(mockException); drainableAppCallback.drain(); verify(mockApp).reportError(eq(YarnTaskSchedulerServiceError.RESOURCEMANAGER_ERROR), argumentCaptor.capture(), any(DagInfo.class)); assertTrue(argumentCaptor.getValue().contains("mockexception")); scheduler.onShutdownRequest(); drainableAppCallback.drain(); verify(mockApp).appShutdownRequested(); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testSimpleReuseLocalMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from w ww. j av a 2 s . c om drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host3"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, hostsv0t0); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, hostsv0t0); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, hostsv0t0); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); taskRequestCaptor.scheduleTask(taskv0t1); TaskRequest reqv0t2 = taskRequestCaptor.scheduleTask(taskv0t2); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // finish v0t0 successfully, verify v0t1 is skipped and v0t2 instead is assigned to the container assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t2.task, taskv0t2.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t2); // finish v0t2 successfully, verify v1t0 is assigned to the same container assertTrue(scheduler.deallocateTask(taskv0t2.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // fail v1t0 and verify container is released instead of reused for v1t1 assertTrue(scheduler.deallocateTask(taskv1t0.task, false, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testSimpleReuseRackMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();//from w w w . ja va2 s .c o m drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host2"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, "host4", "/somerack"); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, "host1"); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, "host5"); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); TaskRequest reqv0t1 = taskRequestCaptor.scheduleTask(taskv0t1); taskRequestCaptor.scheduleTask(taskv0t2); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // finish v0t0 successfully, verify v0t1 is skipped and v1t0 assigned instead // since host locality is preferred to rack locality and lower priority vertex // is not blocked by higher priority vertex assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // finish v1t0 successfully, verify v0t1 is assigned assertTrue(scheduler.deallocateTask(taskv1t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t1.task, taskv0t1.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t1); // fail v0t1 and verify container is released instead of reused for v1t1 assertTrue(scheduler.deallocateTask(taskv0t1.task, false, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }
From source file:org.apache.tez.dag.app.rm.TestDagAwareYarnTaskScheduler.java
License:Apache License
@Test(timeout = 30000) public void testSimpleReuseAnyMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); String appHost = "host"; int appPort = 0; String appUrl = "url"; Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 100); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, true); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, true); conf.setInt(TezConfiguration.TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX, 100); DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(10); when(mockDagInfo.getVertexDescendants(anyInt())).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); MockClock clock = new MockClock(1000); NewTaskSchedulerForTest scheduler = new NewTaskSchedulerForTest(drainableAppCallback, mockRMClient, clock); scheduler.initialize();/*w w w . ja va 2 s .c o m*/ drainableAppCallback.drain(); scheduler.start(); drainableAppCallback.drain(); verify(mockRMClient).start(); verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), regResponse.getApplicationACLs(), regResponse.getClientToAMTokenMasterKey(), regResponse.getQueue()); assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); String[] hostsv0t0 = { "host1", "host2" }; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host2"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, "host4", "/rack4"); MockTaskInfo taskv1t0 = new MockTaskInfo("taskv1t0", priorityv1, "host1"); MockTaskInfo taskv1t1 = new MockTaskInfo("taskv1t1", priorityv1, "host6", "/rack6"); TaskRequestCaptor taskRequestCaptor = new TaskRequestCaptor(mockRMClient, scheduler, drainableAppCallback); TaskRequest reqv0t0 = taskRequestCaptor.scheduleTask(taskv0t0); TaskRequest reqv0t1 = taskRequestCaptor.scheduleTask(taskv0t1); TaskRequest reqv0t2 = taskRequestCaptor.scheduleTask(taskv0t2); TaskRequest reqv1t0 = taskRequestCaptor.scheduleTask(taskv1t0); taskRequestCaptor.scheduleTask(taskv1t1); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); ContainerId cid1 = ContainerId.newContainerId(attemptId, 1); Container container1 = Container.newInstance(cid1, host1, null, taskv0t0.capability, priorityv0, null); // allocate one container at v0 priority scheduler.onContainersAllocated(Collections.singletonList(container1)); drainableAppCallback.drain(); verify(mockApp).taskAllocated(taskv0t0.task, taskv0t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t0); // finish v0t0 successfully, verify v0t1 is skipped and v1t0 assigned instead // since host locality is preferred to rack locality and lower priority vertex // is not blocked by higher priority vertex assertTrue(scheduler.deallocateTask(taskv0t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv1t0.task, taskv1t0.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv1t0); // finish v1t0 successfully, verify v0t1 is assigned assertTrue(scheduler.deallocateTask(taskv1t0.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t1.task, taskv0t1.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t1); // finish v0t1 successfully, verify v0t2 is assigned assertTrue(scheduler.deallocateTask(taskv0t1.task, true, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp, never()).containerBeingReleased(any(ContainerId.class)); verify(mockRMClient, never()).releaseAssignedContainer(any(ContainerId.class)); verify(mockApp).taskAllocated(taskv0t2.task, taskv0t2.cookie, container1); verify(mockRMClient).removeContainerRequest(reqv0t2); // fail v0t2 and verify container is released instead of reused for v1t1 assertTrue(scheduler.deallocateTask(taskv0t2.task, false, null, null)); clock.incrementTime(10000); drainableAppCallback.drain(); verify(mockApp).containerBeingReleased(cid1); verify(mockRMClient).releaseAssignedContainer(cid1); String appMsg = "success"; AppFinalStatus finalStatus = new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient).unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl); verify(mockRMClient).stop(); }