Example usage for org.apache.hadoop.yarn.api.records NodeId getHost

List of usage examples for org.apache.hadoop.yarn.api.records NodeId getHost

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. NodeId
  6. getHost

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records NodeId getHost.

Prototype

@Public
@Stable
public abstract String getHost();

Source Link

Document

Get the hostname of the node.

Usage

From source file:com.cloudera.kitten.appmaster.service.ContainerManagerConnectionFactoryImpl.java

License:Open Source License

@Override
public synchronized ContainerManager connect(Container container) {
    NodeId nodeId = container.getNodeId();
    String containerIpPort = String.format("%s:%d", nodeId.getHost(), nodeId.getPort());
    if (!containerManagers.containsKey(containerIpPort)) {
        LOG.info("Connecting to ContainerManager at: " + containerIpPort);
        InetSocketAddress addr = NetUtils.createSocketAddr(containerIpPort);
        ContainerManager cm = (ContainerManager) rpc.getProxy(ContainerManager.class, addr, conf);
        containerManagers.put(containerIpPort, cm);
        return cm;
    }/*from   w  w  w . j ava  2s. c o m*/
    return containerManagers.get(containerIpPort);
}

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

private Map<String, String> getDataNodeNodeManagerMapping(Configuration conf) throws Exception {
    Map<String, String> map = new HashMap<String, String>();
    DFSClient dfsClient = new DFSClient(new URI(conf.get("fs.defaultFS")), conf);
    DatanodeInfo[] DNs = dfsClient.datanodeReport(HdfsConstants.DatanodeReportType.ALL);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);/*from   w ww .j a  va 2 s . com*/
    yarnClient.start();
    List<NodeId> nodeIds = getYarnNodeIds(conf);
    if (nodeIds.size() != DNs.length) {
        throw new RuntimeException("Number of DNs and NMs differ, MiniLlama "
                + "node mapping requires them to be equal at startup");
    }
    LOG.info("HDFS/YARN mapping:");
    for (int i = 0; i < DNs.length; i++) {
        String key = DNs[i].getXferAddr();
        NodeId nodeId = nodeIds.get(i);
        String value = nodeId.getHost() + ":" + nodeId.getPort();
        map.put(key, value);
        LOG.info("  DN/NM: " + key + "/" + value);
    }
    yarnClient.stop();
    nodes = map.size();
    verifySingleHost(map.keySet(), "DataNode");
    verifySingleHost(map.values(), "NodeManager");
    return map;
}

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

public String getNodeName(NodeId nodeId) {
    return (includePortInNodeName) ? nodeId.getHost() + ":" + nodeId.getPort() : nodeId.getHost();
}

From source file:de.huberlin.wbi.hiway.scheduler.TestScheduler.java

License:Apache License

private static void run(Scheduler scheduler, List<String> nodeNames, List<String> taskNames, int[][] runtimes) {

    Queue<NodeId> availableNodes = new LinkedList<>();
    for (String nodeName : nodeNames) {
        NodeId node = NodeId.newInstance(nodeName, 0);
        availableNodes.add(node);/*from  w  ww.  j a  v  a 2s. c om*/
    }

    Map<Container, TaskInstance> runningTasks = new HashMap<>();
    Map<Container, Long> finishTimes = new HashMap<>();

    int runningId = 0;
    long clock = 0;
    while (!scheduler.nothingToSchedule() || !runningTasks.isEmpty()) {
        if (!scheduler.nothingToSchedule() && !availableNodes.isEmpty()) {
            NodeId nodeId = availableNodes.remove();
            ContainerId containerId = ContainerId.newContainerId(null, runningId++);
            Container container = Container.newInstance(containerId, nodeId, "", null, null, null);
            TaskInstance task = scheduler.getNextTask(container);
            runningTasks.put(container, task);
            long runtime = (runtimes == null) ? 1
                    : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())];

            finishTimes.put(container, clock + runtime);
        }

        for (Container container : finishTimes.keySet()) {
            if (clock == finishTimes.get(container)) {
                NodeId nodeId = container.getNodeId();
                ContainerStatus containerStatus = ContainerStatus.newInstance(container.getId(), null, "", 0);
                TaskInstance task = runningTasks.get(container);
                task.setCompleted();
                long runtime = (runtimes == null) ? 1
                        : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())];
                scheduler.taskCompleted(task, containerStatus, runtime);
                runningTasks.remove(container);
                availableNodes.add(nodeId);
            }
        }

        clock++;
    }
}

From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java

License:Apache License

private synchronized void processAllocation(List<Container> allocatedContainers,
        List<ContainerStatus> completedContainers) {
    System.err.println(allocatedContainers);
    for (Container c : allocatedContainers) {
        System.err.println("Got container: " + c.getContainerStatus());
        NodeId nodeId = c.getNodeId();
        Resource resource = c.getResource();

        Set<AskRecord> arSet = resource2AskMap.get(resource);
        boolean found = false;
        if (arSet != null) {
            AskRecord wildcardMatch = null;
            AskRecord nameMatch = null;//  w  ww  . j av a2  s. co  m
            for (AskRecord ar : arSet) {
                ResourceRequest req = ar.req;
                if (ar.allocation == null) {
                    if ("*".equals(req.getHostName()) && wildcardMatch == null) {
                        wildcardMatch = ar;
                    }
                    if (req.getHostName().equals(nodeId.getHost()) && nameMatch == null) {
                        nameMatch = ar;
                        break;
                    }
                }
            }
            if (nameMatch != null) {
                found = true;
                nameMatch.allocation = c;
            } else if (wildcardMatch != null) {
                found = true;
                wildcardMatch.allocation = c;
            }
        }
        if (!found) {
            System.err.println("Unknown request satisfied: " + resource);
        }
    }
}

From source file:io.hops.util.DBUtility.java

License:Apache License

public static RMNode processHopRMNodeCompsForScheduler(RMNodeComps hopRMNodeComps, RMContext rmContext)
        throws InvalidProtocolBufferException {
    org.apache.hadoop.yarn.api.records.NodeId nodeId;
    RMNode rmNode = null;//from www. j a  v a 2s.co  m
    if (hopRMNodeComps != null) {
        nodeId = ConverterUtils.toNodeId(hopRMNodeComps.getRMNodeId());
        rmNode = rmContext.getRMNodes().get(nodeId);

        // The first time we are receiving the RMNode, this will happen when the node registers
        if (rmNode == null) {
            // Retrieve heartbeat
            boolean nextHeartbeat = true;

            // Create Resource
            Resource resource = null;
            if (hopRMNodeComps.getHopResource() != null) {
                resource = Resource.newInstance(hopRMNodeComps.getHopResource().getMemory(),
                        hopRMNodeComps.getHopResource().getVirtualCores());
            } else {
                LOG.error("ResourceOption should not be null");
                resource = Resource.newInstance(0, 0);
            }
            /*rmNode = new RMNodeImplDist(nodeId, rmContext, hopRMNodeComps.getHopRMNode().getHostName(),
                    hopRMNodeComps.getHopRMNode().getCommandPort(),
                    hopRMNodeComps.getHopRMNode().getHttpPort(),
                    ResourceTrackerService.resolve(hopRMNodeComps.getHopRMNode().getHostName()),
                    resourceOption,
                    hopRMNodeComps.getHopRMNode().getNodemanagerVersion(),
                    hopRMNodeComps.getHopRMNode().getHealthReport(),
                    hopRMNodeComps.getHopRMNode().getLastHealthReportTime(),
                    nextHeartbeat);*/

            rmNode = new RMNodeImplDist(nodeId, rmContext, hopRMNodeComps.getHopRMNode().getHostName(),
                    hopRMNodeComps.getHopRMNode().getCommandPort(), hopRMNodeComps.getHopRMNode().getHttpPort(),
                    ResourceTrackerService.resolve(hopRMNodeComps.getHopRMNode().getHostName()), resource,
                    hopRMNodeComps.getHopRMNode().getNodemanagerVersion());

            // Force Java to put the host in cache
            NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort());
        }

        // Update the RMNode
        if (hopRMNodeComps.getHopRMNode() != null) {
            ((RMNodeImplDist) rmNode).setState(hopRMNodeComps.getHopRMNode().getCurrentState());
        }
        if (hopRMNodeComps.getHopUpdatedContainerInfo() != null) {
            List<io.hops.metadata.yarn.entity.UpdatedContainerInfo> hopUpdatedContainerInfoList = hopRMNodeComps
                    .getHopUpdatedContainerInfo();

            if (hopUpdatedContainerInfoList != null && !hopUpdatedContainerInfoList.isEmpty()) {
                ConcurrentLinkedQueue<org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> updatedContainerInfoQueue = new ConcurrentLinkedQueue<>();

                Map<Integer, org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> ucis = new HashMap<>();
                LOG.debug(hopRMNodeComps.getRMNodeId() + " getting ucis " + hopUpdatedContainerInfoList.size()
                        + " pending event " + hopRMNodeComps.getPendingEvent().getId().getEventId());

                for (io.hops.metadata.yarn.entity.UpdatedContainerInfo hopUCI : hopUpdatedContainerInfoList) {
                    if (!ucis.containsKey(hopUCI.getUpdatedContainerInfoId())) {
                        ucis.put(hopUCI.getUpdatedContainerInfoId(),
                                new org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo(
                                        new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(),
                                        new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(),
                                        hopUCI.getUpdatedContainerInfoId()));
                    }

                    ContainerId cid = ConverterUtils.toContainerId(hopUCI.getContainerId());
                    io.hops.metadata.yarn.entity.ContainerStatus hopContainerStatus = hopRMNodeComps
                            .getHopContainersStatusMap().get(hopUCI.getContainerId());

                    org.apache.hadoop.yarn.api.records.ContainerStatus conStatus = org.apache.hadoop.yarn.api.records.ContainerStatus
                            .newInstance(cid, ContainerState.valueOf(hopContainerStatus.getState()),
                                    hopContainerStatus.getDiagnostics(), hopContainerStatus.getExitstatus());

                    // Check ContainerStatus state to add it in the appropriate list
                    if (conStatus != null) {
                        LOG.debug("add uci for container " + conStatus.getContainerId() + " status "
                                + conStatus.getState());
                        if (conStatus.getState().equals(ContainerState.RUNNING)) {
                            ucis.get(hopUCI.getUpdatedContainerInfoId()).getNewlyLaunchedContainers()
                                    .add(conStatus);
                        } else if (conStatus.getState().equals(ContainerState.COMPLETE)) {
                            ucis.get(hopUCI.getUpdatedContainerInfoId()).getCompletedContainers()
                                    .add(conStatus);
                        }
                    }
                }

                for (org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo uci : ucis
                        .values()) {
                    updatedContainerInfoQueue.add(uci);
                }

                ((RMNodeImplDist) rmNode).setUpdatedContainerInfo(updatedContainerInfoQueue);
            } else {
                LOG.debug(hopRMNodeComps.getRMNodeId()
                        + " hopUpdatedContainerInfoList = null || hopUpdatedContainerInfoList.isEmpty() "
                        + hopRMNodeComps.getPendingEvent().getId().getEventId());
            }
        } else {
            LOG.debug(hopRMNodeComps.getRMNodeId() + " hopRMNodeFull.getHopUpdatedContainerInfo()=null "
                    + hopRMNodeComps.getPendingEvent().getId().getEventId());
        }
    }

    return rmNode;
}

From source file:org.apache.hama.bsp.JobImpl.java

License:Apache License

/**
 *
 * @param rpc// w w  w.  j  a  va2  s  .co  m
 * @param nmToken
 * @param nodeId
 * @param user
 * @return
 */
protected ContainerManagementProtocol getContainerManagementProtocolProxy(final YarnRPC rpc, Token nmToken,
        NodeId nodeId, String user) {
    ContainerManagementProtocol proxy;
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    final InetSocketAddress addr = NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort());
    if (nmToken != null) {
        ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
    }

    proxy = ugi.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
        @Override
        public ContainerManagementProtocol run() {
            return (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, addr, conf);
        }
    });
    return proxy;
}

From source file:org.apache.hoya.yarn.appmaster.state.RoleHistoryUtils.java

License:Apache License

public static String hostnameOf(Container container) {
    NodeId nodeId = container.getNodeId();
    if (nodeId == null) {
        throw new HoyaRuntimeException("Container has no node ID: %s", HoyaUtils.containerToString(container));
    }//from  ww w .  ja  v a 2  s.c  o m
    return nodeId.getHost();
}

From source file:org.apache.myriad.policy.LeastAMNodesFirstPolicy.java

License:Apache License

/**
 * Called whenever a NM HBs to RM. The NM's updates will already be recorded in the
 * SchedulerNode before this method is called.
 *
 * @param event/*from   w  w  w.j a  va2s.  co m*/
 */
private void onNodeUpdated(NodeUpdateSchedulerEvent event) {
    NodeId nodeID = event.getRMNode().getNodeID();
    SchedulerNode schedulerNode = yarnScheduler.getSchedulerNode(nodeID);
    schedulerNodes.put(nodeID.getHost(), schedulerNode); // keep track of only one node per host
}

From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java

License:Apache License

@Override
public CallBackFilter getCallBackFilter() {
    return new CallBackFilter() {
        @Override//w ww  .  j  a  v  a 2 s . c  o  m
        public boolean allowCallBacksForNode(NodeId nodeManager) {
            return SchedulerUtils.isEligibleForFineGrainedScaling(nodeManager.getHost(), state);
        }
    };
}