List of usage examples for org.apache.hadoop.yarn.api.records ResourceOption newInstance
public static ResourceOption newInstance(Resource resource, int overCommitTimeout)
From source file:io.hops.metadata.util.RMUtilities.java
License:Apache License
public static org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode getRMNode(final String id, final RMContext context, final Configuration conf) throws IOException { LightWeightRequestHandler getRMNodeHandler = new LightWeightRequestHandler(YARNOperationType.TEST) { @Override/*w ww. j a va 2s. c o m*/ public Object performTask() throws IOException { connector.beginTransaction(); connector.readLock(); org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode rmNode = null; RMNodeDataAccess rmnodeDA = (RMNodeDataAccess) RMStorageFactory .getDataAccess(RMNodeDataAccess.class); RMNode hopRMNode = (RMNode) rmnodeDA.findByNodeId(id); if (hopRMNode != null) { ResourceDataAccess resDA = (ResourceDataAccess) RMStorageFactory .getDataAccess(ResourceDataAccess.class); NodeDataAccess nodeDA = (NodeDataAccess) RMStorageFactory.getDataAccess(NodeDataAccess.class); //Retrieve resource of RMNode Resource res = (Resource) resDA.findEntry(hopRMNode.getNodeId(), Resource.TOTAL_CAPABILITY, Resource.RMNODE); NodeId nodeId = ConverterUtils.toNodeId(id); //Retrieve and Initialize NodeBase for RMNode org.apache.hadoop.net.Node node = null; if (hopRMNode.getNodeId() != null) { Node hopNode = (Node) nodeDA.findById(hopRMNode.getNodeId()); node = new NodeBase(hopNode.getName(), hopNode.getLocation()); if (hopNode.getParent() != null) { node.setParent(new NodeBase(hopNode.getParent())); } node.setLevel(hopNode.getLevel()); } //Retrieve nextHeartbeat NextHeartbeatDataAccess nextHBDA = (NextHeartbeatDataAccess) RMStorageFactory .getDataAccess(NextHeartbeatDataAccess.class); boolean nextHeartbeat = nextHBDA.findEntry(id); //Create Resource ResourceOption resourceOption = null; if (res != null) { resourceOption = ResourceOption.newInstance(org.apache.hadoop.yarn.api.records.Resource .newInstance(res.getMemory(), res.getVirtualCores()), hopRMNode.getOvercommittimeout()); } rmNode = new RMNodeImpl(nodeId, context, hopRMNode.getHostName(), hopRMNode.getCommandPort(), hopRMNode.getHttpPort(), node, resourceOption, hopRMNode.getNodemanagerVersion(), hopRMNode.getHealthReport(), hopRMNode.getLastHealthReportTime(), nextHeartbeat, conf.getBoolean(YarnConfiguration.HOPS_DISTRIBUTED_RT_ENABLED, YarnConfiguration.DEFAULT_HOPS_DISTRIBUTED_RT_ENABLED)); ((RMNodeImpl) rmNode).setState(hopRMNode.getCurrentState()); // *** Recover maps/lists of RMNode *** //Use a cache for retrieved ContainerStatus Map<String, ContainerStatus> hopContainerStatuses = new HashMap<String, ContainerStatus>(); //1. Recover JustLaunchedContainers JustLaunchedContainersDataAccess jlcDA = (JustLaunchedContainersDataAccess) RMStorageFactory .getDataAccess(JustLaunchedContainersDataAccess.class); ContainerStatusDataAccess containerStatusDA = (ContainerStatusDataAccess) RMStorageFactory .getDataAccess(ContainerStatusDataAccess.class); List<JustLaunchedContainers> hopJlcList = jlcDA.findByRMNode(id); if (hopJlcList != null && !hopJlcList.isEmpty()) { Map<org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.ContainerStatus> justLaunchedContainers = new HashMap<org.apache.hadoop.yarn.api.records.ContainerId, org.apache.hadoop.yarn.api.records.ContainerStatus>(); for (JustLaunchedContainers hop : hopJlcList) { //Create ContainerId org.apache.hadoop.yarn.api.records.ContainerId cid = ConverterUtils .toContainerId(hop.getContainerId()); //Find and create ContainerStatus if (!hopContainerStatuses.containsKey(hop.getContainerId())) { hopContainerStatuses.put(hop.getContainerId(), (ContainerStatus) containerStatusDA.findEntry(hop.getContainerId(), id)); } org.apache.hadoop.yarn.api.records.ContainerStatus conStatus = org.apache.hadoop.yarn.api.records.ContainerStatus .newInstance(cid, ContainerState.valueOf( hopContainerStatuses.get(hop.getContainerId()).getState()), hopContainerStatuses.get(hop.getContainerId()).getDiagnostics(), hopContainerStatuses.get(hop.getContainerId()).getExitstatus()); justLaunchedContainers.put(cid, conStatus); } ((RMNodeImpl) rmNode).setJustLaunchedContainers(justLaunchedContainers); } //2. Return ContainerIdToClean ContainerIdToCleanDataAccess cidToCleanDA = (ContainerIdToCleanDataAccess) RMStorageFactory .getDataAccess(ContainerIdToCleanDataAccess.class); List<ContainerId> cidToCleanList = cidToCleanDA.findByRMNode(id); if (cidToCleanList != null && !cidToCleanList.isEmpty()) { Set<org.apache.hadoop.yarn.api.records.ContainerId> containersToClean = new TreeSet<org.apache.hadoop.yarn.api.records.ContainerId>(); for (ContainerId hop : cidToCleanList) { //Create ContainerId containersToClean.add(ConverterUtils.toContainerId(hop.getContainerId())); } ((RMNodeImpl) rmNode).setContainersToClean(containersToClean); } //3. Finished Applications FinishedApplicationsDataAccess finishedAppsDA = (FinishedApplicationsDataAccess) RMStorageFactory .getDataAccess(FinishedApplicationsDataAccess.class); List<FinishedApplications> hopFinishedAppsList = finishedAppsDA.findByRMNode(id); if (hopFinishedAppsList != null && !hopFinishedAppsList.isEmpty()) { List<ApplicationId> finishedApps = new ArrayList<ApplicationId>(); for (FinishedApplications hop : hopFinishedAppsList) { finishedApps.add(ConverterUtils.toApplicationId(hop.getApplicationId())); } ((RMNodeImpl) rmNode).setFinishedApplications(finishedApps); } //4. UpdadedContainerInfo UpdatedContainerInfoDataAccess uciDA = (UpdatedContainerInfoDataAccess) RMStorageFactory .getDataAccess(UpdatedContainerInfoDataAccess.class); //Retrieve all UpdatedContainerInfo entries for this particular RMNode Map<Integer, List<UpdatedContainerInfo>> hopUpdatedContainerInfoMap = uciDA.findByRMNode(id); if (hopUpdatedContainerInfoMap != null && !hopUpdatedContainerInfoMap.isEmpty()) { ConcurrentLinkedQueue<org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> updatedContainerInfoQueue = new ConcurrentLinkedQueue<org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo>(); for (int uciId : hopUpdatedContainerInfoMap.keySet()) { for (UpdatedContainerInfo hopUCI : hopUpdatedContainerInfoMap.get(uciId)) { List<org.apache.hadoop.yarn.api.records.ContainerStatus> newlyAllocated = new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(); List<org.apache.hadoop.yarn.api.records.ContainerStatus> completed = new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(); //Retrieve containerstatus entries for the particular updatedcontainerinfo org.apache.hadoop.yarn.api.records.ContainerId cid = ConverterUtils .toContainerId(hopUCI.getContainerId()); if (!hopContainerStatuses.containsKey(hopUCI.getContainerId())) { hopContainerStatuses.put(hopUCI.getContainerId(), (ContainerStatus) containerStatusDA.findEntry(hopUCI.getContainerId(), id)); } org.apache.hadoop.yarn.api.records.ContainerStatus conStatus = org.apache.hadoop.yarn.api.records.ContainerStatus .newInstance(cid, ContainerState.valueOf(hopContainerStatuses .get(hopUCI.getContainerId()).getState()), hopContainerStatuses.get(hopUCI.getContainerId()).getDiagnostics(), hopContainerStatuses.get(hopUCI.getContainerId()).getExitstatus()); //Check ContainerStatus state to add it to appropriate list if (conStatus != null) { if (conStatus.getState().toString() .equals(TablesDef.ContainerStatusTableDef.STATE_RUNNING)) { newlyAllocated.add(conStatus); } else if (conStatus.getState().toString() .equals(TablesDef.ContainerStatusTableDef.STATE_COMPLETED)) { completed.add(conStatus); } } org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo uci = new org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo( newlyAllocated, completed, hopUCI.getUpdatedContainerInfoId()); updatedContainerInfoQueue.add(uci); ((RMNodeImpl) rmNode).setUpdatedContainerInfo(updatedContainerInfoQueue); //Update uci counter ((RMNodeImpl) rmNode).setUpdatedContainerInfoId(hopRMNode.getUciId()); } } } //5. Retrieve latestNodeHeartBeatResponse NodeHBResponseDataAccess hbDA = (NodeHBResponseDataAccess) RMStorageFactory .getDataAccess(NodeHBResponseDataAccess.class); NodeHBResponse hopHB = (NodeHBResponse) hbDA.findById(id); if (hopHB != null) { NodeHeartbeatResponse hb = new NodeHeartbeatResponsePBImpl( YarnServerCommonServiceProtos.NodeHeartbeatResponseProto .parseFrom(hopHB.getResponse())); ((RMNodeImpl) rmNode).setLatestNodeHBResponse(hb); } } connector.commit(); return rmNode; } }; return (org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode) getRMNodeHandler.handle(); }
From source file:io.hops.metadata.util.RMUtilities.java
License:Apache License
/** * Recover inactive nodes map of RMContextImpl. * * @param rmContext/*from w ww . j a v a 2 s . c o m*/ * @param state * @return * @throws java.lang.Exception */ //For testing TODO move to test public static Map<String, org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode> getRMContextInactiveNodes( final RMContext rmContext, final RMState state, final Configuration conf) throws Exception { LightWeightRequestHandler getRMContextInactiveNodesHandler = new LightWeightRequestHandler( YARNOperationType.TEST) { @Override public Object performTask() throws StorageException { connector.beginTransaction(); connector.writeLock(); ConcurrentMap<String, org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode> inactiveNodes = new ConcurrentHashMap<String, org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode>(); //Retrieve rmctxnodes table entries RMContextInactiveNodesDataAccess rmctxInactiveNodesDA = (RMContextInactiveNodesDataAccess) RMStorageFactory .getDataAccess(RMContextInactiveNodesDataAccess.class); ResourceDataAccess DA = (ResourceDataAccess) YarnAPIStorageFactory .getDataAccess(ResourceDataAccess.class); RMNodeDataAccess rmDA = (RMNodeDataAccess) RMStorageFactory.getDataAccess(RMNodeDataAccess.class); List<RMContextInactiveNodes> hopRMContextInactiveNodes = rmctxInactiveNodesDA.findAll(); if (hopRMContextInactiveNodes != null && !hopRMContextInactiveNodes.isEmpty()) { for (RMContextInactiveNodes key : hopRMContextInactiveNodes) { NodeId nodeId = ConverterUtils.toNodeId(key.getRmnodeid()); //retrieve RMNode in order to create a new FiCaSchedulerNode RMNode hopRMNode = (RMNode) rmDA.findByNodeId(key.getRmnodeid()); //Retrieve resource of RMNode Resource res = (Resource) DA.findEntry(hopRMNode.getNodeId(), Resource.TOTAL_CAPABILITY, Resource.RMNODE); //Retrieve and Initialize NodeBase for RMNode NodeDataAccess nodeDA = (NodeDataAccess) RMStorageFactory .getDataAccess(NodeDataAccess.class); //Retrieve and Initialize NodeBase for RMNode org.apache.hadoop.net.Node node = null; if (hopRMNode.getNodeId() != null) { Node hopNode = (Node) nodeDA.findById(hopRMNode.getNodeId()); node = new NodeBase(hopNode.getName(), hopNode.getLocation()); if (hopNode.getParent() != null) { node.setParent(new NodeBase(hopNode.getParent())); } node.setLevel(hopNode.getLevel()); } //Retrieve nextHeartbeat NextHeartbeatDataAccess nextHBDA = (NextHeartbeatDataAccess) RMStorageFactory .getDataAccess(NextHeartbeatDataAccess.class); boolean nextHeartbeat = nextHBDA.findEntry(key.getRmnodeid()); org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode rmNode = new RMNodeImpl(nodeId, rmContext, hopRMNode.getHostName(), hopRMNode.getCommandPort(), hopRMNode.getHttpPort(), node, ResourceOption.newInstance(org.apache.hadoop.yarn.api.records.Resource.newInstance( res.getMemory(), res.getVirtualCores()), hopRMNode.getOvercommittimeout()), hopRMNode.getNodemanagerVersion(), hopRMNode.getHealthReport(), hopRMNode.getLastHealthReportTime(), nextHeartbeat, conf.getBoolean(YarnConfiguration.HOPS_DISTRIBUTED_RT_ENABLED, YarnConfiguration.DEFAULT_HOPS_DISTRIBUTED_RT_ENABLED)); ((RMNodeImpl) rmNode).setState(hopRMNode.getCurrentState()); alreadyRecoveredRMContextInactiveNodes.put(rmNode.getNodeID().getHost(), rmNode); inactiveNodes.put(rmNode.getNodeID().getHost(), rmNode); } } connector.commit(); return inactiveNodes; } }; try { if (alreadyRecoveredRMContextInactiveNodes.isEmpty()) { Map<String, org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode> result = (Map<String, org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode>) getRMContextInactiveNodesHandler .handle(); for (org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode node : result.values()) { node.recover(state); } return result; } else { return alreadyRecoveredRMContextInactiveNodes; } } catch (IOException ex) { LOG.error("HOP", ex); } return null; }
From source file:org.apache.myriad.scheduler.fgs.YarnNodeCapacityManager.java
License:Apache License
/** * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity. * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler. * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity. * * @param rmNode/*from ww w.j ava2s .c o m*/ * @param newCapacity */ @SuppressWarnings("unchecked") public void setNodeCapacity(RMNode rmNode, Resource newCapacity) { //NOOP prevent YARN warning changing to same size if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) { return; } if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) { LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID()); return; } yarnSchedulerLock.lock(); try { if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) { Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity); rmNode.getTotalCapability().setMemory(zeroed.getMemory()); rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores()); LOGGER.warn("Asked to set Node {} to a value less than zero! Had {}, setting to {}.", rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString()); } else { rmNode.getTotalCapability().setMemory(newCapacity.getMemory()); rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores()); if (LOGGER.isInfoEnabled()) { LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity); } } // updates the scheduler with the new capacity for the NM. // the event is handled by the scheduler asynchronously rmContext.getDispatcher().getEventHandler() .handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption .newInstance(rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT))); } finally { yarnSchedulerLock.unlock(); } }