List of usage examples for org.apache.hadoop.yarn.server.resourcemanager.rmnode RMNode getNodeID
public NodeId getNodeID();
From source file:io.hops.util.RmStreamingProcessor.java
License:Apache License
private void updateRMContext(RMNode rmNode) { if (LOG.isDebugEnabled()) { LOG.debug("HOP :: PendingEventRetrieval rmNode " + rmNode + ", state: " + rmNode.getState()); }//from w ww . ja v a2s . co m if (rmNode.getState() == NodeState.DECOMMISSIONED || rmNode.getState() == NodeState.REBOOTED || rmNode.getState() == NodeState.LOST) { rmContext.getInactiveRMNodes().put(rmNode.getNodeID(), rmNode); rmContext.getRMNodes().remove(rmNode.getNodeID(), rmNode); } else { rmContext.getInactiveRMNodes().remove(rmNode.getNodeID().getHost(), rmNode); rmContext.getRMNodes().put(rmNode.getNodeID(), rmNode); } }
From source file:io.hops.util.RmStreamingProcessor.java
License:Apache License
private void triggerEvent(final RMNode rmNode, PendingEvent pendingEvent) { if (LOG.isDebugEnabled()) { LOG.debug("NodeUpdate event_pending event trigger event: " + pendingEvent.getId().getEventId() + " : " + pendingEvent.getId().getNodeId()); }/*w w w.ja v a 2s . c o m*/ // TODO Maybe we should put back Hops Global Thread pool exec.submit(new Runnable() { @Override public void run() { NetUtils.normalizeHostName(rmNode.getHostName()); } }); if (pendingEvent.getType().equals(PendingEvent.Type.NODE_ADDED)) { LOG.debug("HOP :: PendingEventRetrieval event NodeAdded: " + pendingEvent); rmContext.getDispatcher().getEventHandler().handle(new NodeAddedSchedulerEvent(rmNode)); } else if (pendingEvent.getType().equals(PendingEvent.Type.NODE_REMOVED)) { LOG.debug("HOP :: PendingEventRetrieval event NodeRemoved: " + pendingEvent); rmContext.getDispatcher().getEventHandler().handle(new NodeRemovedSchedulerEvent(rmNode)); } else if (pendingEvent.getType().equals(PendingEvent.Type.NODE_UPDATED)) { if (pendingEvent.getStatus().equals(PendingEvent.Status.SCHEDULER_FINISHED_PROCESSING)) { LOG.debug("HOP :: NodeUpdate event - event_scheduler - finished_processing RMNode: " + rmNode.getNodeID() + " pending event: " + pendingEvent.getId().getEventId()); rmContext.getDispatcher().getEventHandler().handle(new NodeUpdateSchedulerEvent(rmNode)); } else if (pendingEvent.getStatus().equals(PendingEvent.Status.SCHEDULER_NOT_FINISHED_PROCESSING)) { LOG.debug("NodeUpdate event - event_scheduler - NOT_finished_processing RMNode: " + rmNode.getNodeID() + " pending event: " + pendingEvent.getId().getEventId()); } } }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
@VisibleForTesting protected void handleStatusUpdate(RMNodeEvent event, RMContext context) { if (!(event instanceof RMNodeStatusEvent)) { logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName()); return;//from w w w . j a va 2 s .c o m } RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event; RMNode rmNode = context.getRMNodes().get(event.getNodeId()); String hostName = rmNode.getNodeID().getHost(); Node host = nodeStore.getNode(hostName); if (host != null) { host.snapshotRunningContainers(); } /* * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. * If the sum is greater than the max capacity of the node, reject the offer. */ Resource offeredResources = getNewResourcesOfferedByMesos(hostName); Resource currentResources = getResourcesUnderUse(statusEvent); if (offerWithinResourceLimits(currentResources, offeredResources)) { yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources)); logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory()); } else { logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory()); } }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandlerTest.java
License:Apache License
private RMNodeStatusEvent getRMStatusEvent(RMNode node) { NodeId id = node.getNodeID(); NodeHealthStatus hStatus = NodeHealthStatusPBImpl.newInstance(true, "HEALTHY", System.currentTimeMillis()); List<ContainerStatus> cStatus = Lists.newArrayList(getContainerStatus(node)); List<ApplicationId> keepAliveIds = Lists.newArrayList(getApplicationId(node.getHttpPort())); NodeHeartbeatResponse response = new NodeHeartbeatResponsePBImpl(); return new RMNodeStatusEvent(id, hStatus, cStatus, keepAliveIds, response); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java
License:Apache License
/** * Checks if any containers were allocated in the current scheduler run and * launches the corresponding Mesos tasks. It also updates the node * capacity depending on what portion of the consumed offers were actually * used./*w w w .j av a 2 s . c o m*/ */ @VisibleForTesting protected void handleContainerAllocation(RMNode rmNode) { String host = rmNode.getNodeID().getHost(); ConsumedOffer consumedOffer = offerLifecycleMgr.drainConsumedOffer(host); if (consumedOffer == null) { LOGGER.debug("No offer consumed for {}", host); return; } Node node = nodeStore.getNode(host); Set<RMContainer> containersBeforeSched = node.getContainerSnapshot(); Set<RMContainer> containersAfterSched = new HashSet<>(node.getNode().getRunningContainers()); Set<RMContainer> containersAllocatedByMesosOffer = (containersBeforeSched == null) ? containersAfterSched : Sets.difference(containersAfterSched, containersBeforeSched); if (containersAllocatedByMesosOffer.isEmpty()) { LOGGER.debug("No containers allocated using Mesos offers for host: {}", host); for (Protos.Offer offer : consumedOffer.getOffers()) { offerLifecycleMgr.declineOffer(offer); } decrementNodeCapacity(rmNode, OfferUtils.getYarnResourcesFromMesosOffers(consumedOffer.getOffers())); } else { LOGGER.debug("Containers allocated using Mesos offers for host: {} count: {}", host, containersAllocatedByMesosOffer.size()); // Identify the Mesos tasks that need to be launched List<Protos.TaskInfo> tasks = Lists.newArrayList(); Resource resUsed = Resource.newInstance(0, 0); for (RMContainer newContainer : containersAllocatedByMesosOffer) { tasks.add(getTaskInfoForContainer(newContainer, consumedOffer, node)); resUsed = Resources.add(resUsed, newContainer.getAllocatedResource()); } // Reduce node capacity to account for unused offers Resource resOffered = OfferUtils.getYarnResourcesFromMesosOffers(consumedOffer.getOffers()); Resource resUnused = Resources.subtract(resOffered, resUsed); decrementNodeCapacity(rmNode, resUnused); myriadDriver.getDriver().launchTasks(consumedOffer.getOfferIds(), tasks); } // No need to hold on to the snapshot anymore node.removeContainerSnapshot(); }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java
License:Apache License
/** * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity. * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler. * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity. * * @param rmNode/* w w w. j ava 2 s. c om*/ * @param newCapacity */ @SuppressWarnings("unchecked") public void setNodeCapacity(RMNode rmNode, Resource newCapacity) { //NOOP prevent YARN warning changing to same size if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) { return; } if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) { LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID()); return; } yarnSchedulerLock.lock(); try { if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) { Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity); rmNode.getTotalCapability().setMemory(zeroed.getMemory()); rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores()); LOGGER.warn("Asked to set Node {} to a value less than zero! Had {}, setting to {}.", rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString()); } else { rmNode.getTotalCapability().setMemory(newCapacity.getMemory()); rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores()); if (LOGGER.isInfoEnabled()) { LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity); } } // updates the scheduler with the new capacity for the NM. // the event is handled by the scheduler asynchronously rmContext.getDispatcher().getEventHandler() .handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption .newInstance(rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT))); } finally { yarnSchedulerLock.unlock(); } }
From source file:org.apache.myriad.TestObjectFactory.java
License:Apache License
/** * Returns a new RMContainer corresponding to the RMNode and RMContext. The RMContainer is the * ResourceManager's view of an application container per the Hadoop docs * //from w w w . j a v a2 s. c om * @param node * @param context * @param appId * @param cores * @param memory * @return RMContainer */ public static RMContainer getRMContainer(RMNode node, RMContext context, int appId, int cores, int memory) { ContainerId containerId = ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(123456789, 1), 1), appId); Container container = Container.newInstance(containerId, node.getNodeID(), node.getHttpAddress(), Resources.createResource(memory, cores), null, null); return new RMContainerImpl(container, containerId.getApplicationAttemptId(), node.getNodeID(), "user1", context); }
From source file:org.apache.myriad.TestObjectFactory.java
License:Apache License
public static RMNodeStatusEvent getRMStatusEvent(RMNode node) { NodeId id = node.getNodeID(); NodeHealthStatus hStatus = NodeHealthStatusPBImpl.newInstance(true, "HEALTHY", System.currentTimeMillis()); List<ContainerStatus> cStatus = Lists.newArrayList(getContainerStatus(node)); List<ApplicationId> keepAliveIds = Lists.newArrayList(getApplicationId(node.getHttpPort())); NodeHeartbeatResponse response = new NodeHeartbeatResponsePBImpl(); return new RMNodeStatusEvent(id, hStatus, cStatus, keepAliveIds, response); }