Example usage for org.apache.hadoop.yarn.client.api NMTokenCache NMTokenCache

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api NMTokenCache NMTokenCache.

Prototype

public NMTokenCache() 

Source Link

Document

Creates a NM token cache instance.

Usage

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void _registerSchedulerAndCreateNMClient(String queue) throws Exception {
    NMTokenCache nmTokenCache = new NMTokenCache();
    nmClient = NMClient.createNMClient();
    nmClient.setNMTokenCache(nmTokenCache);
    nmClient.init(yarnConf);// w  w  w  .  jav a  2 s . c o  m
    nmClient.start();
    LOG.debug("Started NMClient, AM '{}' with scheduler for '{}' queue", appId, queue);
    int heartbeatInterval = getConf().getInt(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERNAL_DEFAULT);
    AMRMClient<LlamaContainerRequest> amRmClient = AMRMClient.createAMRMClient();
    amRmClient.setNMTokenCache(nmTokenCache);
    amRmClientAsync = AMRMClientAsync.createAMRMClientAsync(amRmClient, heartbeatInterval,
            YarnRMConnector.this);
    amRmClientAsync.init(yarnConf);
    amRmClientAsync.start();
    String urlWithoutScheme = getConf().get(ADVERTISED_TRACKING_URL_KEY, "http://")
            .substring("http://".length());
    RegisterApplicationMasterResponse response = amRmClientAsync.registerApplicationMaster(
            getConf().get(ADVERTISED_HOSTNAME_KEY, ""), getConf().getInt(ADVERTISED_PORT_KEY, 0),
            urlWithoutScheme);
    maxResource = response.getMaximumResourceCapability();
    nodes = Collections.synchronizedMap(new HashMap<String, Resource>());
    for (NodeReport nodeReport : yarnClient.getNodeReports()) {
        if (nodeReport.getNodeState() == NodeState.RUNNING) {
            String nodeKey = getNodeName(nodeReport.getNodeId());
            nodes.put(nodeKey, nodeReport.getCapability());
            LOG.debug("Added node '{}' with '{}' cpus and '{}' memory", nodeKey,
                    nodeReport.getCapability().getVirtualCores(), nodeReport.getCapability().getMemory());
        }
    }
    LOG.debug("Registered with scheduler, AM '{}' for '{}' queue", appId, queue);
}

---

From source file:org.apache.hama.bsp.JobImpl.java

License:Apache License

@Override
public JobState startJob() throws Exception {

    this.allocatedContainers = new ArrayList<Container>(numBSPTasks);
    NMTokenCache nmTokenCache = new NMTokenCache();
    while (allocatedContainers.size() < numBSPTasks) {
        AllocateRequest req = AllocateRequest.newInstance(lastResponseID, 0.0f,
                createBSPTaskRequest(numBSPTasks - allocatedContainers.size(), taskMemoryInMb, priority),
                releasedContainers, null);

        AllocateResponse allocateResponse = resourceManager.allocate(req);
        for (NMToken token : allocateResponse.getNMTokens()) {
            nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
        }//from w  ww .  j  a v  a  2s  . c o  m

        LOG.info("Got response ID: " + allocateResponse.getResponseId() + " with num of containers: "
                + allocateResponse.getAllocatedContainers().size() + " and following resources: "
                + allocateResponse.getAvailableResources().getMemory() + "mb");
        this.lastResponseID = allocateResponse.getResponseId();

        this.allocatedContainers.addAll(allocateResponse.getAllocatedContainers());

        LOG.info("Waiting to allocate " + (numBSPTasks - allocatedContainers.size()) + " more containers...");

        Thread.sleep(1000l);
    }

    LOG.info("Got " + allocatedContainers.size() + " containers!");

    int id = 0;
    for (Container allocatedContainer : allocatedContainers) {
        LOG.info("Launching task on a new container." + ", containerId=" + allocatedContainer.getId()
                + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":"
                + allocatedContainer.getNodeId().getPort() + ", containerNodeURI="
                + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory"
                + allocatedContainer.getResource().getMemory());

        // Connect to ContainerManager on the allocated container
        String user = conf.get("bsp.user.name");
        if (user == null) {
            user = System.getenv(ApplicationConstants.Environment.USER.name());
        }

        ContainerManagementProtocol cm = null;
        try {
            cm = getContainerManagementProtocolProxy(yarnRPC,
                    nmTokenCache.getToken(allocatedContainer.getNodeId().toString()),
                    allocatedContainer.getNodeId(), user);
        } catch (Exception e) {
            LOG.error("Failed to create ContainerManager...");
            if (cm != null)
                yarnRPC.stopProxy(cm, conf);
            e.printStackTrace();
        }

        BSPTaskLauncher runnableLaunchContainer = new BSPTaskLauncher(id, allocatedContainer, cm, conf, jobFile,
                jobId);

        launchers.put(id, runnableLaunchContainer);
        runnableLaunchContainer.start();
        completionQueue.add(runnableLaunchContainer);
        id++;
    }

    LOG.info("Waiting for tasks to finish...");
    state = JobState.RUNNING;
    int completed = 0;

    List<Integer> cleanupTasks = new ArrayList<Integer>();
    while (completed != numBSPTasks) {
        for (BSPTaskLauncher task : completionQueue) {
            BSPTaskStatus returnedTask = task.poll();
            // if our task returned with a finished state
            if (returnedTask != null) {
                if (returnedTask.getExitStatus() != 0) {
                    LOG.error("Task with id \"" + returnedTask.getId() + "\" failed!");
                    cleanupTask(returnedTask.getId());
                    state = JobState.FAILED;
                    return state;
                } else {
                    LOG.info("Task \"" + returnedTask.getId() + "\" sucessfully finished!");
                    completed++;
                    LOG.info("Waiting for " + (numBSPTasks - completed) + " tasks to finish!");
                }
                cleanupTasks.add(returnedTask.getId());
            }
        }
        Thread.sleep(1000L);
    }

    for (Integer stopId : cleanupTasks) {
        cleanupTask(stopId);
    }

    state = JobState.SUCCESS;
    return state;
}

---

From source file:org.elasticsearch.hadoop.yarn.am.ApplicationMaster.java

License:Apache License

void run() {
    log.info("Starting ApplicationMaster...");

    if (nmTokenCache == null) {
        nmTokenCache = new NMTokenCache();
    }//from   w w  w .j a  v  a2  s. c o  m

    if (rpc == null) {
        rpc = new AppMasterRpc(cfg, nmTokenCache);
        rpc.start();
    }

    // register AM
    appId = YarnUtils.getApplicationAttemptId(env);
    Assert.notNull(appId, "ApplicationAttemptId cannot be found in env %s" + env);
    RegisterApplicationMasterResponse amResponse = rpc.registerAM();
    cluster = new EsCluster(rpc, appConfig, env);

    try {
        cluster.start();
    } finally {
        close();
    }
}

---

From source file:org.springframework.yarn.support.compat.NMTokenCacheCompat.java

License:Apache License

/**
 * Gets the Hadoop {@code NMTokenCache}. Access token cache via singleton
 * method if exists, otherwise assumes that class is hadoop vanilla
 * implementation with static methods where we can just use default
 * constructor and further access for those static method would
 * got via instance itself./*from  w  w  w .  j a va  2s  .  c om*/
 * <p>
 * This is due to a fact that i.e. cloudera modified this class by
 * removing static methods and made access to it via singleton pattern.
 *
 * @return the token cache
 */
public static NMTokenCache getNMTokenCache() {
    if (nmTokenCache == null) {
        synchronized (lock) {
            if (nmTokenCache == null) {
                Method singletonMethod = ReflectionUtils.findMethod(NMTokenCache.class, "getSingleton");
                if (singletonMethod != null) {
                    if (log.isDebugEnabled()) {
                        log.debug("Creating NMTokenCache using NMTokenCache.getSingleton()");
                    }
                    nmTokenCache = (NMTokenCache) ReflectionUtils.invokeMethod(singletonMethod, null);
                } else {
                    log.debug(
                            "Creating NMTokenCache using constructor, further access via instance static methods.");
                    nmTokenCache = new NMTokenCache();
                }
            }
        }
    }
    return nmTokenCache;
}

---