Example usage for org.apache.hadoop.yarn.api.records Container getId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getId.

Prototype

@Public
@Stable
public abstract ContainerId getId();

Source Link

Document

Get the globally unique identifier for the container.

Usage

From source file:ApplicationMaster.java

License:Apache License

private static void publishContainerStartEvent(TimelineClient timelineClient, Container container)
        throws IOException, YarnException {
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(container.getId().toString());
    entity.setEntityType(DSEntity.DS_CONTAINER.toString());
    entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser().getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(DSEvent.DS_CONTAINER_START.toString());
    event.addEventInfo("Node", container.getNodeId().toString());
    event.addEventInfo("Resources", container.getResource().toString());
    entity.addEvent(event);/*ww w .  ja  va 2 s. c  o m*/

    timelineClient.putEntities(entity);
}

---

From source file:alluxio.yarn.ApplicationMaster.java

License:Apache License

private void launchMasterContainer(Container container) {
    String command = YarnUtils.buildCommand(YarnContainerType.ALLUXIO_MASTER);
    try {/*from  w w  w.  j  a v a 2 s.c om*/
        ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
        ctx.setCommands(Lists.newArrayList(command));
        ctx.setLocalResources(setupLocalResources(mResourcePath));
        ctx.setEnvironment(setupMasterEnvironment());
        if (UserGroupInformation.isSecurityEnabled()) {
            ctx.setTokens(mAllTokens.duplicate());
        }
        LOG.info("Launching container {} for Alluxio master on {} with master command: {}", container.getId(),
                container.getNodeHttpAddress(), command);
        mNMClient.startContainer(container, ctx);
        String containerUri = container.getNodeHttpAddress(); // in the form of 1.2.3.4:8042
        mMasterContainerNetAddress = containerUri.split(":")[0];
        LOG.info("Master address: {}", mMasterContainerNetAddress);
        return;
    } catch (Exception e) {
        LOG.error("Error launching container {}", container.getId(), e);
    }
}

---

From source file:alluxio.yarn.ApplicationMaster.java

License:Apache License

private void launchWorkerContainer(Container container) {
    String command = YarnUtils.buildCommand(YarnContainerType.ALLUXIO_WORKER);

    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    ctx.setCommands(Lists.newArrayList(command));
    ctx.setLocalResources(setupLocalResources(mResourcePath));
    ctx.setEnvironment(setupWorkerEnvironment(mMasterContainerNetAddress, mRamdiskMemInMB));
    if (UserGroupInformation.isSecurityEnabled()) {
        ctx.setTokens(mAllTokens.duplicate());
    }//from w  w w.ja va  2s  .c om

    try {
        LOG.info("Launching container {} for Alluxio worker on {} with worker command: {}", container.getId(),
                container.getNodeHttpAddress(), command);
        mNMClient.startContainer(container, ctx);
    } catch (Exception e) {
        LOG.error("Error launching container {}", container.getId(), e);
    }
}

---

From source file:alluxio.yarn.ContainerAllocator.java

License:Apache License

/**
 * @param container the container which has been allocated by YARN
 *///w  w  w . j a  v a 2  s .  c o  m
public synchronized void allocateContainer(Container container) {
    String containerHost = container.getNodeId().getHost();
    if (mAllocatedContainerHosts.count(containerHost) < mMaxContainersPerHost
            && mAllocatedContainerHosts.size() < mTargetNumContainers) {
        mAllocatedContainerHosts.add(containerHost);
        mAllocatedContainers.add(container);
    } else {
        LOG.info("Releasing assigned container on host {}", containerHost);
        mRMClient.releaseAssignedContainer(container.getId());
    }
    mOutstandingContainerRequestsLatch.countDown();
}

---

From source file:com.cloudera.kitten.appmaster.service.ApplicationMasterServiceImpl1.java

License:Open Source License

@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
    LOG.info("Allocating " + allocatedContainers.size() + " container(s)");
    Set<Container> assigned = Sets.newHashSet();
    for (ContainerTracker tracker : trackers) {
        for (Container allocated : allocatedContainers) {
            if (tracker.needsContainers()) {
                if (!assigned.contains(allocated) && tracker.matches(allocated)) {
                    tracker.launchContainer(allocated);
                    assigned.add(allocated);
                    containerAllocation.put(allocated.getId(), tracker);
                }// w w  w  . j  a v  a  2 s.  co m
            }
        }
    }
    if (assigned.size() < allocatedContainers.size()) {
        LOG.error(String.format("Not all containers were allocated (%d out of %d)", assigned.size(),
                allocatedContainers.size()));
        stop();
    }
}

---

From source file:com.cloudera.kitten.appmaster.service.WorkflowService.java

License:Open Source License

@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
    LOG.info("Allocating " + allocatedContainers.size() + " container(s)");
    Set<Container> assigned = Sets.newHashSet();
    for (ContainerTracker tracker : trackers.values()) {
        for (Container allocated : allocatedContainers) {
            if (tracker.isInitilized && tracker.needsContainers()) {
                if (!assigned.contains(allocated) && tracker.matches(allocated)) {
                    LOG.info("Allocated cores: " + allocated.getResource().getVirtualCores());
                    tracker.launchContainer(allocated);
                    assigned.add(allocated);
                    containerAllocation.put(allocated.getId(), tracker);
                }/*from   w w  w. j  a  va  2s .c  om*/
            }
        }
    }
    for (Entry<ContainerId, ContainerTracker> e : containerAllocation.entrySet()) {
        LOG.info("Allocated: " + e.getKey() + " to operator: " + e.getValue().params.getName());
    }
    /*if (assigned.size() < allocatedContainers.size()) {
      LOG.error(String.format("Not all containers were allocated (%d out of %d)", assigned.size(),
          allocatedContainers.size()));
      stop();
    }*/
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void _release(Collection<RMResource> resources) throws LlamaException {
    for (RMResource resource : resources) {
        YarnRMConnector connector = (YarnRMConnector) resource.getRmData().get(YARN_RM_CONNECTOR_KEY);
        if (connector == null || !connector.equals(this)) {
            LOG.warn("Resource being released is not from this connector. "
                    + "Resource [{}], Resource connector [{}], this [{}]", resource, connector, this);
            continue; // Not allocated by this connector.
        }//from w ww . j a va  2  s.co  m

        boolean released = false;
        LlamaContainerRequest request = (LlamaContainerRequest) resource.getRmData().get("request");
        if (request != null) {
            LOG.debug("Releasing container request for '{}'", resource);
            amRmClientAsync.removeContainerRequest(request);
            released = true;
        }
        Container container = (Container) resource.getRmData().get("container");
        if (container != null) {
            LOG.debug("Releasing container '{}' for '{}'", container, resource);
            containerToResourceMap.remove(container.getId());
            queue(new ContainerHandler(ugi, resource, container, Action.STOP));
            released = true;
        } else {
            LOG.debug("Container was not allocated yet for '{}'.", resource);
        }
        if (!released) {
            LOG.debug("Missing RM payload, ignoring release of container " + "request for '{}'", resource);
        }
    }
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private RMEvent createResourceAllocation(RMResource resources, Container container) {
    return RMEvent.createAllocationEvent(resources.getResourceId(), getNodeName(container.getNodeId()),
            container.getResource().getVirtualCores(), container.getResource().getMemory(), container.getId(),
            resources.getRmData());/*from w  w  w  .  j  a v a2 s. c o m*/
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void handleContainerMatchingRequest(Container container, LlamaContainerRequest req,
        List<RMEvent> changes) {
    RMResource resource = req.getResourceAsk();

    LOG.debug("New allocation for '{}' container '{}', node '{}'", resource, container.getId(),
            container.getNodeId());/*from  www .  j  a  v  a2s  . c  o  m*/

    resource.getRmData().put("container", container);
    containerToResourceMap.put(container.getId(), resource.getResourceId());
    changes.add(createResourceAllocation(resource, container));
    amRmClientAsync.removeContainerRequest(req);
    LOG.trace("Reservation resource '{}' removed from YARN", resource);

    queue(new ContainerHandler(ugi, resource, container, Action.START));
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public void onContainersAllocated(List<Container> containers) {
    List<RMEvent> changes = new ArrayList<RMEvent>();
    // no need to use a ugi.doAs() as this is called from within Yarn client
    List<Container> unclaimedContainers = new ArrayList<Container>();
    for (Container container : containers) {
        List<? extends Collection<LlamaContainerRequest>> matchingContainerReqs = amRmClientAsync
                .getMatchingRequests(container.getPriority(), getNodeName(container.getNodeId()),
                        container.getResource());

        if (!matchingContainerReqs.isEmpty()) {
            LlamaContainerRequest req = null;
            Iterator<? extends Collection<LlamaContainerRequest>> it1 = matchingContainerReqs.iterator();
            while (req == null && it1.hasNext()) {
                Iterator<LlamaContainerRequest> it2 = it1.next().iterator();
                while (req == null && it2.hasNext()) {
                    req = it2.next();//from   w w  w  . jav a  2 s .  c  om
                    LOG.trace("Matching container '{}' resource '{}'", container, req.getResourceAsk());
                }
            }
            if (req == null) {
                LOG.error("There was a match for container '{}', " + "LlamaContainerRequest cannot be NULL",
                        container);
            } else {
                handleContainerMatchingRequest(container, req, changes);
                /*Remove the granted request from anyLocationResourceIdToRequestMap if it is there*/
                anyLocationResourceIdToRequestMap.remove(req.getResourceAsk().getResourceId());
            }
        } else {
            LOG.debug("No strong request match for {}. Adding to the list of unclaimed containers.", container);
            unclaimedContainers.add(container);
        }
    }
    /*Matching YARN resources against requests relaxing locality*/
    for (Container container : unclaimedContainers) {
        /*Looking for requests with 'DONT_CARE' or 'PREFERRED' locality which match with the resources we've got*/
        boolean containerIsClaimed = false;
        Iterator<Map.Entry<UUID, LlamaContainerRequest>> iterator = anyLocationResourceIdToRequestMap.entrySet()
                .iterator();
        while (iterator.hasNext()) {
            Map.Entry<UUID, LlamaContainerRequest> entry = iterator.next();
            LlamaContainerRequest request = entry.getValue();
            /*Matching by the capacity only*/
            if (request.getResourceAsk().getCpuVCoresAsk() == container.getResource().getVirtualCores()
                    && request.getResourceAsk().getMemoryMbsAsk() == container.getResource().getMemory()) {
                handleContainerMatchingRequest(container, request, changes);
                iterator.remove();
                containerIsClaimed = true;
                break;
            }
        }
        if (!containerIsClaimed) {
            LOG.error("No matching request for {}. Releasing the container.", container);
            containerToResourceMap.remove(container.getId());
            amRmClientAsync.releaseAssignedContainer(container.getId());
        }
    }
    llamaCallback.onEvent(changes);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• »