List of usage examples for org.apache.hadoop.yarn.api.records Resource newInstance
@Public
@Stable
public static Resource newInstance(long memory, int vCores)
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@VisibleForTesting protected Resource getResourcesUnderUse(RMNodeStatusEvent statusEvent) { Resource usedResources = Resource.newInstance(0, 0); for (ContainerStatus status : statusEvent.getContainers()) { if (containerInUse(status)) { RMContainer rmContainer = yarnScheduler.getRMContainer(status.getContainerId()); // (sdaingade) This check is needed as RMContainer information may not be populated // immediately after a RM restart. if (rmContainer != null) { Resources.addTo(usedResources, rmContainer.getAllocatedResource()); }/*from w ww . j a v a2 s . co m*/ } } return usedResources; }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandlerTest.java
License:Apache License
@Override @Before/*from www . ja v a 2 s . c o m*/ public void setUp() throws Exception { super.setUp(); this.baseStateStoreDirectory = "/tmp/nm-heartbeat-handler-test"; context = new MockRMContext(); context.setDispatcher(TestObjectFactory.getMockDispatcher()); context.setSystemMetricsPublisher(new SystemMetricsPublisher()); profileZero = TestObjectFactory.getServiceResourceProfile("zero", Double.valueOf(0.0), Double.valueOf(0.0), Long.valueOf(0), Long.valueOf(0)); profileSmall = TestObjectFactory.getServiceResourceProfile("small", Double.valueOf(2.0), Double.valueOf(2048.0), Long.valueOf(1), Long.valueOf(1024)); nodeOne = TestObjectFactory.getRMNode("localhost-one", 8800, Resource.newInstance(0, 0)); nodeTwo = TestObjectFactory.getRMNode("localhost-two", 8800, Resource.newInstance(1024, 2)); sNodeOne = new FSSchedulerNode(nodeOne, false); sNodeTwo = new FSSchedulerNode(nodeTwo, false); nodeTaskOne = TestObjectFactory.getNodeTask("localhost-one", profileZero); nodeTaskTwo = TestObjectFactory.getNodeTask("localhost-two", profileSmall); ConcurrentMap<NodeId, RMNode> rmNodes = new ConcurrentHashMap<NodeId, RMNode>(); rmNodes.put(nodeOne.getNodeID(), nodeOne); rmNodes.put(nodeTwo.getNodeID(), nodeTwo); context.setRMNodes(rmNodes); store = new NodeStore(); store.add(sNodeOne); store.add(sNodeTwo); MyriadDriver driver = TestObjectFactory.getMyriadDriver(new MockSchedulerDriver()); olManager = new OfferLifecycleManager(store, driver); state = TestObjectFactory.getSchedulerState(new MyriadConfiguration(), "/tmp/nm-heartbeat-handler-test"); state.addNodes(Lists.newArrayList(nodeTaskOne, nodeTaskTwo)); MyriadFairScheduler scheduler = TestObjectFactory.getMyriadFairScheduler(context); scheduler.addNode(sNodeOne); scheduler.addNode(sNodeTwo); manager = new YarnNodeCapacityManager(new CompositeInterceptor(), scheduler, context, driver, olManager, store, state, new TaskUtils(this.cfg)); handler = new NMHeartBeatHandler(new CompositeInterceptor(), scheduler, driver, manager, olManager, store, state, cfg.getNodeManagerConfiguration()); }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandlerTest.java
License:Apache License
@Test public void testOfferWithinResourceLimits() throws Exception { Resource resourcesOne = Resource.newInstance(512, 1); Resource offerOne = Resource.newInstance(1024, 2); Resource offerTwo = Resource.newInstance(4096, 2); Resource offerThree = Resource.newInstance(1024, 8); assertTrue(handler.offerWithinResourceLimits(resourcesOne, offerOne)); assertFalse(handler.offerWithinResourceLimits(resourcesOne, offerTwo)); assertFalse(handler.offerWithinResourceLimits(resourcesOne, offerThree)); }
From source file:org.apache.myriad.scheduler.fgs.NodeTest.java
License:Apache License
@Override @Before//from w ww. j a v a 2 s. c o m public void setUp() throws Exception { super.setUp(); context = new MockRMContext(); context.setDispatcher(TestObjectFactory.getMockDispatcher()); context.setSystemMetricsPublisher(new SystemMetricsPublisher()); nodeOne = TestObjectFactory.getRMNode("localhost-one", 8800, Resource.newInstance(1024, 2)); nodeTwo = TestObjectFactory.getRMNode("localhost-two", 8800, Resource.newInstance(2048, 4)); sNodeOne = new FSSchedulerNode(nodeOne, false); sNodeTwo = new FSSchedulerNode(nodeTwo, false); store = new NodeStore(); store.add(sNodeOne); store.add(sNodeTwo); containerOne = TestObjectFactory.getRMContainer(nodeOne, context, 1, 2, 1024); }
From source file:org.apache.myriad.scheduler.fgs.OfferUtils.java
License:Apache License
/** * Transforms a collection of mesos offers into {@link Resource}. * * @param offers collection of mesos offers * @return a single resource object equivalent to the cumulative sum of mesos offers *//*from ww w. j av a2 s . c o m*/ public static Resource getYarnResourcesFromMesosOffers(Collection<Offer> offers) { double cpus = 0.0; double mem = 0.0; for (Protos.Offer offer : offers) { for (Protos.Resource resource : offer.getResourcesList()) { if (resource.getName().equalsIgnoreCase("cpus")) { cpus += resource.getScalar().getValue(); } else if (resource.getName().equalsIgnoreCase("mem")) { mem += resource.getScalar().getValue(); } } } return Resource.newInstance((int) mem, (int) cpus); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java
License:Apache License
/** * Checks if any containers were allocated in the current scheduler run and * launches the corresponding Mesos tasks. It also updates the node * capacity depending on what portion of the consumed offers were actually * used./*from w w w . j a v a 2s .c om*/ */ @VisibleForTesting protected void handleContainerAllocation(RMNode rmNode) { String host = rmNode.getNodeID().getHost(); ConsumedOffer consumedOffer = offerLifecycleMgr.drainConsumedOffer(host); if (consumedOffer == null) { LOGGER.debug("No offer consumed for {}", host); return; } Node node = nodeStore.getNode(host); Set<RMContainer> containersBeforeSched = node.getContainerSnapshot(); Set<RMContainer> containersAfterSched = new HashSet<>(node.getNode().getRunningContainers()); Set<RMContainer> containersAllocatedByMesosOffer = (containersBeforeSched == null) ? containersAfterSched : Sets.difference(containersAfterSched, containersBeforeSched); if (containersAllocatedByMesosOffer.isEmpty()) { LOGGER.debug("No containers allocated using Mesos offers for host: {}", host); for (Protos.Offer offer : consumedOffer.getOffers()) { offerLifecycleMgr.declineOffer(offer); } decrementNodeCapacity(rmNode, OfferUtils.getYarnResourcesFromMesosOffers(consumedOffer.getOffers())); } else { LOGGER.debug("Containers allocated using Mesos offers for host: {} count: {}", host, containersAllocatedByMesosOffer.size()); // Identify the Mesos tasks that need to be launched List<Protos.TaskInfo> tasks = Lists.newArrayList(); Resource resUsed = Resource.newInstance(0, 0); for (RMContainer newContainer : containersAllocatedByMesosOffer) { tasks.add(getTaskInfoForContainer(newContainer, consumedOffer, node)); resUsed = Resources.add(resUsed, newContainer.getAllocatedResource()); } // Reduce node capacity to account for unused offers Resource resOffered = OfferUtils.getYarnResourcesFromMesosOffers(consumedOffer.getOffers()); Resource resUnused = Resources.subtract(resOffered, resUsed); decrementNodeCapacity(rmNode, resUnused); myriadDriver.getDriver().launchTasks(consumedOffer.getOfferIds(), tasks); } // No need to hold on to the snapshot anymore node.removeContainerSnapshot(); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManagerTest.java
License:Apache License
@Override @Before/*from ww w. j a v a 2 s . c o m*/ public void setUp() throws Exception { super.setUp(); this.baseStateStoreDirectory = "/tmp/yarn-node-capacity-manager-test"; context = new MockRMContext(); context.setDispatcher(TestObjectFactory.getMockDispatcher()); context.setSystemMetricsPublisher(new SystemMetricsPublisher()); nodeOne = TestObjectFactory.getRMNode("localhost-one", 8800, Resource.newInstance(2048, 4)); nodeTwo = TestObjectFactory.getRMNode("localhost-two", 8800, Resource.newInstance(1024, 2)); sNodeOne = new FSSchedulerNode(nodeOne, false); sNodeTwo = new FSSchedulerNode(nodeTwo, false); containerOne = TestObjectFactory.getRMContainer(nodeOne, context, 1, 2, 1024); store = new NodeStore(); store.add(sNodeOne); store.add(sNodeTwo); MyriadDriver driver = TestObjectFactory.getMyriadDriver(new MockSchedulerDriver()); olManager = new OfferLifecycleManager(store, driver); state = TestObjectFactory.getSchedulerState(new MyriadConfiguration(), "/tmp/yarn-node-capacity-manager-test"); MyriadFairScheduler scheduler = TestObjectFactory.getMyriadFairScheduler(context); scheduler.addNode(sNodeOne); scheduler.addNode(sNodeTwo); manager = new YarnNodeCapacityManager(new CompositeInterceptor(), scheduler, context, driver, olManager, store, state, new TaskUtils(this.cfg)); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManagerTest.java
License:Apache License
@Test public void testIncrementNodeCapacity() throws Exception { manager.incrementNodeCapacity(nodeTwo, Resource.newInstance(2048, 4)); assertEquals(3072, nodeTwo.getTotalCapability().getMemory()); assertEquals(6, nodeTwo.getTotalCapability().getVirtualCores()); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManagerTest.java
License:Apache License
@Test public void testDecrementNodeCapacity() throws Exception { manager.decrementNodeCapacity(nodeOne, Resource.newInstance(1024, 2)); assertEquals(1024, nodeOne.getTotalCapability().getMemory()); assertEquals(2, nodeOne.getTotalCapability().getVirtualCores()); }
From source file:org.apache.myriad.scheduler.ResourceUtils.java
License:Apache License
public static Resource componentwiseMax(Resource lhs, Resource rhs) { int cores = Math.max(lhs.getVirtualCores(), rhs.getVirtualCores()); int mem = Math.max(lhs.getMemory(), rhs.getMemory()); return Resource.newInstance(cores, mem); }