Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:ApplicationMaster.java

License:Apache License

private static void publishContainerStartEvent(TimelineClient timelineClient, Container container)
        throws IOException, YarnException {
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(container.getId().toString());
    entity.setEntityType(DSEntity.DS_CONTAINER.toString());
    entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser().getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(DSEvent.DS_CONTAINER_START.toString());
    event.addEventInfo("Node", container.getNodeId().toString());
    event.addEventInfo("Resources", container.getResource().toString());
    entity.addEvent(event);/* w w  w  .j  av  a2s.  c o m*/

    timelineClient.putEntities(entity);
}

---

From source file:alluxio.yarn.ApplicationMasterTest.java

License:Apache License

/**
 * Mocks mRMClient to randomly allocated one of the requested hosts.
 *
 * This involves/*from  ww w.  ja v a  2 s . c  o m*/
 * 1) Creating NUM_WORKERS mock containers, each with a different mock host
 * 2) Mocking mYarnClient to return the mock hosts of the mock containers
 * 3) Mocking mRMClient.addContainerRequest to asynchronously call mMaster.onContainersAllocated
 * with a random container on a requested host
 *
 * @param numContainers the number of mock container hosts
 */
private void mockResourceManager(int numContainers) throws Exception {
    final Random random = new Random();
    final List<Container> mockContainers = Lists.newArrayList();
    List<NodeReport> nodeReports = Lists.newArrayList();
    List<String> hosts = Lists.newArrayList(MASTER_ADDRESS);
    for (int i = 0; i < numContainers - 1; i++) {
        String host = "host" + i;
        hosts.add(host);
    }
    for (String host : hosts) {
        Container mockContainer = Mockito.mock(Container.class);
        Mockito.when(mockContainer.getNodeHttpAddress()).thenReturn(host + ":8042");
        Mockito.when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance(host, 0));
        mockContainers.add(mockContainer);
        NodeReport report = Mockito.mock(NodeReport.class);
        Mockito.when(report.getNodeId()).thenReturn(NodeId.newInstance(host, 0));
        nodeReports.add(report);
    }
    // We need to use anyVararg because Mockito is dumb and assumes that an array argument must be
    // vararg. Using regular any() will only match an array if it has length 1.
    Mockito.when(mYarnClient.getNodeReports(Matchers.<NodeState[]>anyVararg())).thenReturn(nodeReports);

    // Pretend to be the Resource Manager, allocating containers when they are requested.
    Mockito.doAnswer(new Answer<Void>() {
        @Override
        public Void answer(final InvocationOnMock invocation) {
            new Thread(new Runnable() {
                @Override
                public void run() {
                    // Allow the requests to interleave randomly
                    CommonUtils.sleepMs(50 + random.nextInt(200));
                    // Allocate a randomly chosen container from among the requested hosts
                    ContainerRequest request = invocation.getArgumentAt(0, ContainerRequest.class);
                    Set<String> requestedHosts = Sets.newHashSet(request.getNodes());
                    List<Container> requestedContainers = Lists.newArrayList();
                    for (Container container : mockContainers) {
                        if (requestedHosts.contains(container.getNodeId().getHost())) {
                            requestedContainers.add(container);
                        }
                    }
                    mMaster.onContainersAllocated(Lists
                            .newArrayList(requestedContainers.get(random.nextInt(requestedContainers.size()))));
                }
            }).start();
            return null;
        }
    }).when(mRMClient).addContainerRequest(Mockito.<ContainerRequest>any());
}

---

From source file:alluxio.yarn.ApplicationMasterTest.java

License:Apache License

/**
 * Tests that the Alluxio worker containers are launched properly.
 */// www  .  ja  va2 s.co  m
@Test
public void launchAlluxioWorkerContainersTest() throws Exception {
    Container mockContainer1 = Mockito.mock(Container.class);
    Container mockContainer2 = Mockito.mock(Container.class);
    // The containers must be from different hosts because we don't support multiple clients on the
    // same host.
    Mockito.when(mockContainer1.getNodeId()).thenReturn(NodeId.newInstance("host1", 0));
    Mockito.when(mockContainer2.getNodeId()).thenReturn(NodeId.newInstance("host2", 0));
    // Say that the master is allocated so that container offers are assumed to be worker offers
    mPrivateAccess.getMasterAllocated().countDown();
    mPrivateAccess.setMasterContainerAddress("masterAddress");
    mPrivateAccess.setOutstandingWorkerContainerRequestsLatch(new CountDownLatch(2));

    List<Container> containers = Lists.newArrayList(mockContainer1, mockContainer2);

    mMaster.onContainersAllocated(containers);

    // Generate the context that we expect Yarn to launch workers with.
    Map<String, String> expectedWorkerEnvironment = ImmutableMap.<String, String>builder()
            .put("ALLUXIO_HOME", ApplicationConstants.Environment.PWD.$())
            .put("ALLUXIO_MASTER_HOSTNAME", "masterAddress")
            .put("ALLUXIO_WORKER_MEMORY_SIZE", Integer.toString(RAMDISK_MEM_MB) + ".00MB").build();
    String expectedWorkerCommand = "./alluxio-yarn-setup.sh alluxio-worker 1><LOG_DIR>/stdout 2><LOG_DIR>/stderr ";
    ContainerLaunchContext expectedWorkerContext = ContainerLaunchContext.newInstance(
            getExpectedLocalResources(), expectedWorkerEnvironment, Lists.newArrayList(expectedWorkerCommand),
            null, null, null);

    Mockito.verify(mNMClient).startContainer(Mockito.same(mockContainer1),
            Mockito.argThat(getContextMatcher(expectedWorkerContext)));
    Mockito.verify(mNMClient).startContainer(Mockito.same(mockContainer2),
            Mockito.argThat(getContextMatcher(expectedWorkerContext)));
    Assert.assertEquals(containers.size(), mPrivateAccess.getWorkerHosts().size());
}

---

From source file:alluxio.yarn.ContainerAllocator.java

License:Apache License

/**
 * @param container the container which has been allocated by YARN
 *//*from w w w .  j  a va 2s.  c om*/
public synchronized void allocateContainer(Container container) {
    String containerHost = container.getNodeId().getHost();
    if (mAllocatedContainerHosts.count(containerHost) < mMaxContainersPerHost
            && mAllocatedContainerHosts.size() < mTargetNumContainers) {
        mAllocatedContainerHosts.add(containerHost);
        mAllocatedContainers.add(container);
    } else {
        LOG.info("Releasing assigned container on host {}", containerHost);
        mRMClient.releaseAssignedContainer(container.getId());
    }
    mOutstandingContainerRequestsLatch.countDown();
}

---

From source file:alluxio.yarn.ContainerAllocatorTest.java

License:Apache License

private void testFullAllocation(int numHosts, int maxContainersPerHost) throws Exception {
    int numContainers = numHosts * maxContainersPerHost;
    ContainerAllocator containerAllocator = setup(numHosts, maxContainersPerHost, numContainers);
    List<Container> containers = containerAllocator.allocateContainers();

    Set<String> containerHosts = new HashSet<>();
    for (Container container : containers) {
        containerHosts.add(container.getNodeId().getHost());
    }/*from  w w  w . jav  a 2s  .c  om*/
    assertEquals("All hosts are allocated", numHosts, containerHosts.size());
    assertEquals("All containers are allocated", numContainers, containers.size());
    checkMaxHostsLimitNotExceeded(containers, maxContainersPerHost);
}

---

From source file:alluxio.yarn.ContainerAllocatorTest.java

License:Apache License

private void checkMaxHostsLimitNotExceeded(List<Container> containers, int maxContainersPerHost) {
    ConcurrentHashMap<String, Integer> counts = new ConcurrentHashMap<>();
    for (Container container : containers) {
        String host = container.getNodeId().getHost();
        counts.putIfAbsent(host, 0);//from   w w  w .jav  a  2s.c  o  m
        int newCount = counts.get(host) + 1;
        assertTrue(newCount <= maxContainersPerHost);
        counts.put(host, newCount);
    }
}

---

From source file:com.cloudera.kitten.appmaster.service.ContainerManagerConnectionFactoryImpl.java

License:Open Source License

@Override
public synchronized ContainerManager connect(Container container) {
    NodeId nodeId = container.getNodeId();
    String containerIpPort = String.format("%s:%d", nodeId.getHost(), nodeId.getPort());
    if (!containerManagers.containsKey(containerIpPort)) {
        LOG.info("Connecting to ContainerManager at: " + containerIpPort);
        InetSocketAddress addr = NetUtils.createSocketAddr(containerIpPort);
        ContainerManager cm = (ContainerManager) rpc.getProxy(ContainerManager.class, addr, conf);
        containerManagers.put(containerIpPort, cm);
        return cm;
    }/*from w  w  w.  jav a2 s .  co m*/
    return containerManagers.get(containerIpPort);
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private RMEvent createResourceAllocation(RMResource resources, Container container) {
    return RMEvent.createAllocationEvent(resources.getResourceId(), getNodeName(container.getNodeId()),
            container.getResource().getVirtualCores(), container.getResource().getMemory(), container.getId(),
            resources.getRmData());//  www.j  av  a 2s  .  c o  m
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private void handleContainerMatchingRequest(Container container, LlamaContainerRequest req,
        List<RMEvent> changes) {
    RMResource resource = req.getResourceAsk();

    LOG.debug("New allocation for '{}' container '{}', node '{}'", resource, container.getId(),
            container.getNodeId());

    resource.getRmData().put("container", container);
    containerToResourceMap.put(container.getId(), resource.getResourceId());
    changes.add(createResourceAllocation(resource, container));
    amRmClientAsync.removeContainerRequest(req);
    LOG.trace("Reservation resource '{}' removed from YARN", resource);

    queue(new ContainerHandler(ugi, resource, container, Action.START));
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public void onContainersAllocated(List<Container> containers) {
    List<RMEvent> changes = new ArrayList<RMEvent>();
    // no need to use a ugi.doAs() as this is called from within Yarn client
    List<Container> unclaimedContainers = new ArrayList<Container>();
    for (Container container : containers) {
        List<? extends Collection<LlamaContainerRequest>> matchingContainerReqs = amRmClientAsync
                .getMatchingRequests(container.getPriority(), getNodeName(container.getNodeId()),
                        container.getResource());

        if (!matchingContainerReqs.isEmpty()) {
            LlamaContainerRequest req = null;
            Iterator<? extends Collection<LlamaContainerRequest>> it1 = matchingContainerReqs.iterator();
            while (req == null && it1.hasNext()) {
                Iterator<LlamaContainerRequest> it2 = it1.next().iterator();
                while (req == null && it2.hasNext()) {
                    req = it2.next();//  www  .jav  a2 s . com
                    LOG.trace("Matching container '{}' resource '{}'", container, req.getResourceAsk());
                }
            }
            if (req == null) {
                LOG.error("There was a match for container '{}', " + "LlamaContainerRequest cannot be NULL",
                        container);
            } else {
                handleContainerMatchingRequest(container, req, changes);
                /*Remove the granted request from anyLocationResourceIdToRequestMap if it is there*/
                anyLocationResourceIdToRequestMap.remove(req.getResourceAsk().getResourceId());
            }
        } else {
            LOG.debug("No strong request match for {}. Adding to the list of unclaimed containers.", container);
            unclaimedContainers.add(container);
        }
    }
    /*Matching YARN resources against requests relaxing locality*/
    for (Container container : unclaimedContainers) {
        /*Looking for requests with 'DONT_CARE' or 'PREFERRED' locality which match with the resources we've got*/
        boolean containerIsClaimed = false;
        Iterator<Map.Entry<UUID, LlamaContainerRequest>> iterator = anyLocationResourceIdToRequestMap.entrySet()
                .iterator();
        while (iterator.hasNext()) {
            Map.Entry<UUID, LlamaContainerRequest> entry = iterator.next();
            LlamaContainerRequest request = entry.getValue();
            /*Matching by the capacity only*/
            if (request.getResourceAsk().getCpuVCoresAsk() == container.getResource().getVirtualCores()
                    && request.getResourceAsk().getMemoryMbsAsk() == container.getResource().getMemory()) {
                handleContainerMatchingRequest(container, request, changes);
                iterator.remove();
                containerIsClaimed = true;
                break;
            }
        }
        if (!containerIsClaimed) {
            LOG.error("No matching request for {}. Releasing the container.", container);
            containerToResourceMap.remove(container.getId());
            amRmClientAsync.releaseAssignedContainer(container.getId());
        }
    }
    llamaCallback.onEvent(changes);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »