List of usage examples for org.apache.hadoop.yarn.api.records Resource getMemory
@Public @Deprecated public abstract int getMemory();
From source file:org.apache.metron.maas.service.ContainerTracker.java
License:Apache License
public BlockingQueue<Container> getQueue(Resource resource) { synchronized (acceptedContainersByResource) { int key = getAdjustedSize(resource.getMemory()); BlockingQueue<Container> queue = acceptedContainersByResource.get(key); if (queue == null) { queue = new LinkedBlockingDeque<>(); acceptedContainersByResource.put(key, queue); }/*from w w w. j a va 2s. c o m*/ return queue; } }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@Override public void beforeRMNodeEventHandled(RMNodeEvent event, RMContext context) { switch (event.getType()) { case STARTED: // Since the RMNode was just started, it should not have a non-zero capacity RMNode rmNode = context.getRMNodes().get(event.getNodeId()); if (isNonZeroCapacityNode(rmNode)) { Resource totalCapability = rmNode.getTotalCapability(); logger.warn(//from w w w . j ava2 s.co m "FineGrainedScaling feature got invoked for a NM with non-zero capacity. Host: {}, Mem: {}, CPU: {}. Setting the " + "NM's capacity to (0G,0CPU)", rmNode.getHostName(), totalCapability.getMemory(), totalCapability.getVirtualCores()); totalCapability.setMemory(0); totalCapability.setVirtualCores(0); } break; case STATUS_UPDATE: handleStatusUpdate(event, context); break; default: break; } }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@VisibleForTesting protected boolean isNonZeroCapacityNode(RMNode node) { Resource resource = node.getTotalCapability(); return (resource.getMemory() != 0 || resource.getVirtualCores() != 0); }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@VisibleForTesting protected void handleStatusUpdate(RMNodeEvent event, RMContext context) { if (!(event instanceof RMNodeStatusEvent)) { logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName()); return;//from w w w . j a va 2 s.c o m } RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event; RMNode rmNode = context.getRMNodes().get(event.getNodeId()); String hostName = rmNode.getNodeID().getHost(); Node host = nodeStore.getNode(hostName); if (host != null) { host.snapshotRunningContainers(); } /* * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. * If the sum is greater than the max capacity of the node, reject the offer. */ Resource offeredResources = getNewResourcesOfferedByMesos(hostName); Resource currentResources = getResourcesUnderUse(statusEvent); if (offerWithinResourceLimits(currentResources, offeredResources)) { yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources)); logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory()); } else { logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory()); } }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@VisibleForTesting protected boolean offerWithinResourceLimits(Resource currentResources, Resource offeredResources) { int newMemory = currentResources.getMemory() + offeredResources.getMemory(); int newCores = currentResources.getVirtualCores() + offeredResources.getVirtualCores(); return (newMemory <= conf.getJvmMaxMemoryMB() && newCores <= conf.getMaxCpus()); }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@VisibleForTesting protected Resource getNewResourcesOfferedByMesos(String hostname) { OfferFeed feed = offerLifecycleMgr.getOfferFeed(hostname); List<Offer> offers = new ArrayList<>(); Protos.Offer offer;//from w ww . java2s. c o m while ((offer = feed.poll()) != null) { offers.add(offer); offerLifecycleMgr.markAsConsumed(offer); } Resource fromMesosOffers = OfferUtils.getYarnResourcesFromMesosOffers(offers); if (logger.isDebugEnabled()) { logger.debug("NM on host {} got {} CPUs and {} memory from mesos", hostname, fromMesosOffers.getVirtualCores(), fromMesosOffers.getMemory()); } return fromMesosOffers; }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandlerTest.java
License:Apache License
@Test public void testGetNewResourcesOfferedByMesos() throws Exception { Offer offerOne = TestObjectFactory.getOffer("localhost-one", "slave-one", "mock", "offer-one", 1.0, 512.0); Offer offerTwo = TestObjectFactory.getOffer("localhost-two", "slave-two", "mock", "offer-two", 2.0, 1024.0); olManager.addOffers(offerOne);/* w ww. j ava 2s . c o m*/ olManager.addOffers(offerTwo); Resource resourcesOne = handler.getNewResourcesOfferedByMesos("localhost-one"); assertEquals(1.0, resourcesOne.getVirtualCores(), 0.0); assertEquals(512.0, resourcesOne.getMemory(), 0.0); Resource resourcesTwo = handler.getNewResourcesOfferedByMesos("localhost-two"); assertEquals(2.0, resourcesTwo.getVirtualCores(), 0.0); assertEquals(1024.0, resourcesTwo.getMemory(), 0.0); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java
License:Apache License
private void removeYarnTask(RMContainer rmContainer) { if (containersNotNull(rmContainer)) { Protos.TaskID taskId = containerToTaskId(rmContainer); /*/*from www. j av a2 s.c om*/ * Mark the task as killable within the ServerState object to flag the task * for the TaskTerminator daemon to kill the task */ state.makeTaskKillable(taskId); Node node = retrieveNode(rmContainer); if (node != null) { RMNode rmNode = node.getNode().getRMNode(); Resource resource = rmContainer.getContainer().getResource(); decrementNodeCapacity(rmNode, resource); LOGGER.info("Removed task yarn_{} with exit status freeing {} cpu and {} mem.", rmContainer.getContainer().toString(), rmContainer.getContainerExitStatus(), resource.getVirtualCores(), resource.getMemory()); } else { LOGGER.warn("The Node for the {} host was not found", rmContainer.getContainer().getNodeId().getHost()); } } }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java
License:Apache License
/** * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity. * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler. * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity. * * @param rmNode//from www .jav a 2 s.co m * @param newCapacity */ @SuppressWarnings("unchecked") public void setNodeCapacity(RMNode rmNode, Resource newCapacity) { //NOOP prevent YARN warning changing to same size if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) { return; } if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) { LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID()); return; } yarnSchedulerLock.lock(); try { if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) { Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity); rmNode.getTotalCapability().setMemory(zeroed.getMemory()); rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores()); LOGGER.warn("Asked to set Node {} to a value less than zero! Had {}, setting to {}.", rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString()); } else { rmNode.getTotalCapability().setMemory(newCapacity.getMemory()); rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores()); if (LOGGER.isInfoEnabled()) { LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity); } } // updates the scheduler with the new capacity for the NM. // the event is handled by the scheduler asynchronously rmContext.getDispatcher().getEventHandler() .handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption .newInstance(rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT))); } finally { yarnSchedulerLock.unlock(); } }
From source file:org.apache.myriad.scheduler.ResourceUtils.java
License:Apache License
public static Resource componentwiseMax(Resource lhs, Resource rhs) { int cores = Math.max(lhs.getVirtualCores(), rhs.getVirtualCores()); int mem = Math.max(lhs.getMemory(), rhs.getMemory()); return Resource.newInstance(cores, mem); }