List of usage examples for org.apache.hadoop.yarn.server.resourcemanager MockNM getNodeId
public NodeId getNodeId()
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Test(timeout = 60000) public void test() throws Exception { MockRM rm = new MockRM(conf); rm.start();/*from w ww . j ava 2s. c o m*/ //register two NodeManagers MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); Thread.sleep(1000); //verify that they are persisted in the db List<FiCaSchedulerNode> dbNodes = RMUtilities.getAllFiCaSchedulerNodes(); assertEquals(2, dbNodes.size()); //submit an application of 2GB memory RMApp app1 = rm.submitApp(2048); Thread.sleep(2000); //at this point tables : //ha_fifoscheduler_apps, //ha_schedulerapplication //should container one row for the specific app. You can verify that looking at the db. //also the tables : //ha_appschedulinginfo, RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); //get the scheduler FifoScheduler fifoScheduler = (FifoScheduler) rm.getResourceScheduler(); //get applications map Map<ApplicationId, org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication> apps = fifoScheduler .getSchedulerApplications(); //get nodes map Map<NodeId, org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode> nodes = fifoScheduler .getNodes(); //get current application org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication sA = apps .get(app1.getApplicationId()); //get currentApplicationAttemptId SchedulerApplicationAttempt saAttempt = sA.getCurrentAppAttempt(); List<org.apache.hadoop.yarn.api.records.ResourceRequest> reqs = saAttempt.getAppSchedulingInfo() .getAllResourceRequests(); //retrieve requests from the database Map<String, List<ResourceRequest>> requests = RMUtilities.getAllResourceRequests(); List<ResourceRequest> dbReqs = requests.get(attempt1.getAppAttemptId().toString()); //compare assertEquals(reqs.size(), dbReqs.size()); //its time to kick the scheduling, 2GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); //Thread.sleep(1000); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = fifoScheduler.getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); //retrieve used Resource from database Resource resource = RMUtilities.getResource(nm1.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); assertEquals(2 * GB, resource.getMemory()); //get newlyAllocatedContainers List<RMContainer> newlyAllocatedContainers = saAttempt.getNewlyAllocatedContainers(); //get launchedContainers org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode ficaNode = nodes .get(nm1.getNodeId()); List<RMContainer> launchedContainers = ficaNode.getRunningContainers(); //get liveContainers Map<ContainerId, RMContainer> liveContainers = saAttempt.getLiveContainersMap(); //retrieve newlyAllocatedContainers from the database List<FiCaSchedulerAppNewlyAllocatedContainers> dbNewlyAlCont = RMUtilities .getNewlyAllocatedContainers(attempt1.getAppAttemptId().toString()); //retrieve launchedContainers from the database Map<String, List<LaunchedContainers>> map = RMUtilities.getAllLaunchedContainers(); List<LaunchedContainers> dbLaunchCont = map.get(nm1.getNodeId().toString()); //retrieve liveContainers from the database Map<String, List<FiCaSchedulerAppLiveContainers>> mapLiveCont = RMUtilities.getAllLiveContainers(); List<FiCaSchedulerAppLiveContainers> dbLiveCont = mapLiveCont.get(attempt1.getAppAttemptId().toString()); assertEquals(newlyAllocatedContainers.size(), dbNewlyAlCont.size()); assertEquals(launchedContainers.size(), dbLaunchCont.size()); assertEquals(liveContainers.size(), dbLiveCont.size()); //submit a second application of 2GB memory RMApp app2 = rm.submitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); SchedulerNodeReport report_nm2 = fifoScheduler.getNodeReport(nm2.getNodeId()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); //retrieve used Resource from database Resource hopResource2 = RMUtilities.getResource(nm2.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); assertEquals(2 * GB, hopResource2.getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // add request for containers am2.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 0, 1); AllocateResponse alloc2Response = am2.schedule(); // send the request // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0 GB to nm1 nm1.nodeHeartbeat(true); //Thread.sleep(1000); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(1000); alloc1Response = am1.schedule(); } while (alloc2Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 2..."); Thread.sleep(1000); alloc2Response = am2.schedule(); } // kick the scheduler, nothing given remaining 2 GB to nm2 nm2.nodeHeartbeat(true); Thread.sleep(1000); List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); List<Container> allocated2 = alloc2Response.getAllocatedContainers(); Assert.assertEquals(1, allocated2.size()); Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemory()); Resource nm1AvailableResource = RMUtilities.getResource(nm1.getNodeId().toString(), Resource.AVAILABLE, Resource.FICASCHEDULERNODE); Resource nm2AvailableResource = RMUtilities.getResource(nm2.getNodeId().toString(), Resource.AVAILABLE, Resource.FICASCHEDULERNODE); assertEquals(0, nm1AvailableResource.getMemory()); assertEquals(2 * GB, nm2AvailableResource.getMemory()); Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); Resource nm1UsedResource = RMUtilities.getResource(nm1.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); Resource nm2UsedResource = RMUtilities.getResource(nm2.getNodeId().toString(), Resource.USED, Resource.FICASCHEDULERNODE); assertEquals(6 * GB, nm1UsedResource.getMemory()); assertEquals(2 * GB, nm2UsedResource.getMemory()); Thread.sleep(2000); rm.stop(); Thread.sleep(2000); }