List of usage examples for org.apache.hadoop.yarn.server.resourcemanager MockRM registerNode
public MockNM registerNode(String nodeIdStr, Resource nodeCapability) throws Exception
From source file:io.hops.metadata.util.TestFairSchedulerUtilities.java
License:Apache License
@Test public void TestSimpleFairShareCalculation() throws Exception { MockRM rm = new MockRM(conf); rm.start();/*from w ww . j a v a2 s. c o m*/ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.1:1234", 6 * GB); //submit an application of 2GB memory RMApp app1 = rm.submitApp(3 * GB, "", "user1", null, "queue1"); RMApp app2 = rm.submitApp(1 * GB, "", "user2", null, "queue2"); nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); nm1.nodeHeartbeat(true); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); Thread.sleep(3000); //get the Scheduler FairScheduler fairScheduler = (FairScheduler) rm.getResourceScheduler(); Collection<FSLeafQueue> queues = fairScheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); rm.stop(); }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 30000) public void random() throws Exception { MockRM rm = new MockRM(conf); rm.start();//from www . java 2 s.com //register one NodeManager MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); //submit an application of 2GB memory RMApp app1 = rm.submitApp(3 * GB, "", "user1", null, "queue1"); RMApp app2 = rm.submitApp(3 * GB, "", "user2", null, "queue2"); /** * ************************ * THIS TEST PASSES AS IT IS IF WE INCREASE THE MEMORY OF ONE OF THE 2 APPS, * THEN IT WILL FAIL AS THE * ENTIRE CLUSTER CAPACITY IS 6 GB * APPATTEMPT STATE WILL NOT BE ALLOCATED AND AN EXCEPTION WILL BE THROWN ************************* */ nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 60000) public void test() throws Exception { MockRM rm = new MockRM(conf); rm.start();//ww w . ja v a2 s . com //register two NodeManagers MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); Thread.sleep(1000); //verify that they are persisted in the db List<FiCaSchedulerNode> dbNodes = RMUtilities.getAllFiCaSchedulerNodes(); assertEquals(2, dbNodes.size()); //submit an application of 2GB memory RMApp app1 = rm.submitApp(2048); Thread.sleep(2000); //at this point tables : //ha_fifoscheduler_apps, //ha_schedulerapplication //should container one row for the specific app. You can verify that looking at the db. //also the tables : //ha_appschedulinginfo, RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); //get the scheduler FifoScheduler fifoScheduler = (FifoScheduler) rm.getResourceScheduler(); //get applications map Map<ApplicationId, org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication> apps = fifoScheduler .getSchedulerApplications(); //get nodes map Map<NodeId, org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode> nodes = fifoScheduler .getNodes(); //get current application org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication sA = apps .get(app1.getApplicationId()); //get currentApplicationAttemptId SchedulerApplicationAttempt saAttempt = sA.getCurrentAppAttempt(); List<org.apache.hadoop.yarn.api.records.ResourceRequest> reqs = saAttempt.getAppSchedulingInfo() .getAllResourceRequests(); //retrieve requests from the database Map<String, List<ResourceRequest>> requests = RMUtilities.getAllResourceRequests(); List<ResourceRequest> dbReqs = requests.get(attempt1.getAppAttemptId().toString()); //compare assertEquals(reqs.size(), dbReqs.size()); //its time to kick the scheduling, 2GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); //Thread.sleep(1000); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = fifoScheduler.getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); //retrieve used Resource from database Resource resource = RMUtilities.getResource(nm1.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); assertEquals(2 * GB, resource.getMemory()); //get newlyAllocatedContainers List<RMContainer> newlyAllocatedContainers = saAttempt.getNewlyAllocatedContainers(); //get launchedContainers org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode ficaNode = nodes .get(nm1.getNodeId()); List<RMContainer> launchedContainers = ficaNode.getRunningContainers(); //get liveContainers Map<ContainerId, RMContainer> liveContainers = saAttempt.getLiveContainersMap(); //retrieve newlyAllocatedContainers from the database List<FiCaSchedulerAppNewlyAllocatedContainers> dbNewlyAlCont = RMUtilities .getNewlyAllocatedContainers(attempt1.getAppAttemptId().toString()); //retrieve launchedContainers from the database Map<String, List<LaunchedContainers>> map = RMUtilities.getAllLaunchedContainers(); List<LaunchedContainers> dbLaunchCont = map.get(nm1.getNodeId().toString()); //retrieve liveContainers from the database Map<String, List<FiCaSchedulerAppLiveContainers>> mapLiveCont = RMUtilities.getAllLiveContainers(); List<FiCaSchedulerAppLiveContainers> dbLiveCont = mapLiveCont.get(attempt1.getAppAttemptId().toString()); assertEquals(newlyAllocatedContainers.size(), dbNewlyAlCont.size()); assertEquals(launchedContainers.size(), dbLaunchCont.size()); assertEquals(liveContainers.size(), dbLiveCont.size()); //submit a second application of 2GB memory RMApp app2 = rm.submitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); SchedulerNodeReport report_nm2 = fifoScheduler.getNodeReport(nm2.getNodeId()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); //retrieve used Resource from database Resource hopResource2 = RMUtilities.getResource(nm2.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); assertEquals(2 * GB, hopResource2.getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // add request for containers am2.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 0, 1); AllocateResponse alloc2Response = am2.schedule(); // send the request // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0 GB to nm1 nm1.nodeHeartbeat(true); //Thread.sleep(1000); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(1000); alloc1Response = am1.schedule(); } while (alloc2Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 2..."); Thread.sleep(1000); alloc2Response = am2.schedule(); } // kick the scheduler, nothing given remaining 2 GB to nm2 nm2.nodeHeartbeat(true); Thread.sleep(1000); List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); List<Container> allocated2 = alloc2Response.getAllocatedContainers(); Assert.assertEquals(1, allocated2.size()); Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemory()); Resource nm1AvailableResource = RMUtilities.getResource(nm1.getNodeId().toString(), Resource.AVAILABLE, Resource.FICASCHEDULERNODE); Resource nm2AvailableResource = RMUtilities.getResource(nm2.getNodeId().toString(), Resource.AVAILABLE, Resource.FICASCHEDULERNODE); assertEquals(0, nm1AvailableResource.getMemory()); assertEquals(2 * GB, nm2AvailableResource.getMemory()); Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); Resource nm1UsedResource = RMUtilities.getResource(nm1.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); Resource nm2UsedResource = RMUtilities.getResource(nm2.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); assertEquals(6 * GB, nm1UsedResource.getMemory()); assertEquals(2 * GB, nm2UsedResource.getMemory()); Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 60000) public void testAppSubmissionAndNodeUpdate() throws Exception { MockRM rm = new MockRM(conf); rm.start();// ww w.j a va 2 s.c o m ClientRMService rmService = rm.getClientRMService(); GetApplicationsRequest getRequest = GetApplicationsRequest .newInstance(EnumSet.of(YarnApplicationState.KILLED)); ApplicationId appId1 = getApplicationId(100); ApplicationId appId2 = getApplicationId(101); ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class); when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true); SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null); SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, null, null); try { rmService.submitApplication(submitRequest1); rmService.submitApplication(submitRequest2); } catch (YarnException e) { Assert.fail("Exception is not expected."); } assertEquals("Incorrect number of apps in the RM", 0, rmService.getApplications(getRequest).getApplicationList().size()); Thread.sleep(1000); //test persistance of schedulerapplication Map<String, SchedulerApplication> schedulerApplications = RMUtilities.getSchedulerApplications(); assertEquals("db does not contain good number of schedulerApplications", 2, schedulerApplications.size()); MockNM nm1 = rm.registerNode("host1:1234", 5120); MockNM nm2 = rm.registerNode("host2:5678", 10240); NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertEquals(4000, nodeHeartbeat.getNextHeartBeatInterval()); NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true); Assert.assertEquals(4000, nodeHeartbeat2.getNextHeartBeatInterval()); Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }