Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:org.springframework.yarn.am.allocate.DefaultAllocateCountTracker.java

License:Apache License

public Container processAllocatedContainer(Container container) {
    String host = container.getNodeId().getHost();

    if (modifyWithKey(requestedHosts, host, false)) {
        // match hosts
        log.debug("Found reservation match from hosts for " + host);
    } else if (modifyWithKey(requestedRacks, host, false)) {
        // match racks
        log.debug("Found reservation match from racks for " + host);
    } else if (modify(requestedAny, false)) {
        // match anys
        log.debug("Found reservation match from anys for " + host);
    } else if (decrement(requestedHosts)) {
        // no match - just start to flush out outstanding requests
        log.debug("No reservation match for " + host + ", decremented hosts");
    } else if (decrement(requestedRacks)) {
        // no match - just start to flush out outstanding requests
        log.debug("No reservation match for " + host + ", decremented racks");
    } else if (decrement(requestedAny)) {
        // no match - just start to flush out outstanding requests
        log.debug("No reservation match for " + host + ", decremented anys");
    } else {//from w w  w  .  ja va  2 s . co  m
        // no outstanding requests - mark as garbage
        log.debug("No outstanding requests, marking as garbage");
        return null;
    }

    return container;
}

---

From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitorTests.java

License:Apache License

/**
 * Mock {@link Container}//from  w  ww.j  av a  2s .c  o  m
 *
 * @param containerId the {@link ContainerId}
 * @param nodeId the {@link NodeId}
 * @param resource the {@link Resource}
 * @param priority the {@link Priority}
 * @param state the {@link ContainerState}
 * @return mocked {@link Container}
 */
public static Container getMockContainer(ContainerId containerId, NodeId nodeId, Resource resource,
        Priority priority) {
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(containerId);
    when(container.getNodeId()).thenReturn(nodeId);
    when(container.getResource()).thenReturn(resource);
    when(container.getPriority()).thenReturn(priority);
    return container;
}

---

From source file:org.springframework.yarn.batch.am.AbstractBatchAppmaster.java

License:Apache License

@Override
protected void onContainerAllocated(Container container) {
    if (log.isDebugEnabled()) {
        log.debug("Container allocated: " + container);
    }/*from w  w w. jav a2  s .  c  o m*/

    StepExecution stepExecution = null;

    String host = container.getNodeId().getHost();
    String rack = RackResolver.resolve(host).getNetworkLocation();
    if (log.isDebugEnabled()) {
        log.debug("Matching agains: host=" + host + " rack=" + rack);
    }

    Iterator<Entry<StepExecution, ContainerRequestHint>> iterator = requestData.entrySet().iterator();
    while (iterator.hasNext() && stepExecution != null) {
        Entry<StepExecution, ContainerRequestHint> entry = iterator.next();
        if (entry.getValue() != null && entry.getValue().getHosts() != null) {
            for (String h : entry.getValue().getHosts()) {
                if (h.equals(host)) {
                    stepExecution = entry.getKey();
                    break;
                }
            }
        }
    }

    log.debug("stepExecution after hosts match: " + stepExecution);

    iterator = requestData.entrySet().iterator();
    while (iterator.hasNext() && stepExecution != null) {
        Entry<StepExecution, ContainerRequestHint> entry = iterator.next();
        if (entry.getValue() != null && entry.getValue().getRacks() != null) {
            for (String r : entry.getValue().getRacks()) {
                if (r.equals(rack)) {
                    stepExecution = entry.getKey();
                    break;
                }
            }
        }
    }

    log.debug("stepExecution after racks match: " + stepExecution);

    try {
        if (stepExecution == null) {
            stepExecution = requestData.entrySet().iterator().next().getKey();
        }
        requestData.remove(stepExecution);
        containerToStepMap.put(container.getId(), stepExecution);
        getLauncher().launchContainer(container, getCommands());
    } catch (NoSuchElementException e) {
        log.error("We didn't have step execution in request map.", e);
    }
}

---

From source file:org.springframework.yarn.boot.MockUtils.java

License:Apache License

public static Container getMockContainer(ContainerId containerId, NodeId nodeId, Resource resource,
        Priority priority) {/*from  w  w w  .ja  v a 2 s  .  c om*/
    Container container = mock(Container.class);
    when(container.getId()).thenReturn(containerId);
    when(container.getNodeId()).thenReturn(nodeId);
    when(container.getResource()).thenReturn(resource);
    when(container.getPriority()).thenReturn(priority);
    return container;
}

---

From source file:org.springframework.yarn.examples.AbstractManagedContainerGroupsAppmaster.java

License:Apache License

@Override
protected void onContainerAllocated(Container container) {
    log.info("XXX onContainerAllocated: " + container.getId() + " host=" + container.getNodeId().getHost());
    managedGroups.addContainerNode(new DefaultYarnContainerNode(container));
    getMonitor().addContainer(container);
    getLauncher().launchContainer(container, getCommands());
}

---

From source file:org.springframework.yarn.examples.grid.yarn.GenericContainerGroupResolver.java

License:Apache License

@Override
public List<String> resolveGroupNames(Container container) {
    String containerHost = container.getNodeId().getHost();
    String rack = resolveRacks ? RackResolver.resolve(containerHost).getNetworkLocation() : null;
    ArrayList<String> found = new ArrayList<String>();
    for (Entry<String, List<String>> entry : resolves.entrySet()) {
        for (String host : entry.getValue()) {
            if (safeMatch(containerHost, host) || safeMatch(rack, host)) {
                found.add(entry.getKey());
                break;
            }//from   w ww  .j  av  a 2 s .co  m
        }
    }
    return found;
}

---

From source file:tachyon.yarn.ApplicationMasterTest.java

License:Apache License

/**
 * Mocks mRMClient to randomly allocated one of the requested hosts.
 *
 * This involves/*from   w  ww . j  a v  a2 s .c  om*/
 * 1) Creating NUM_WORKERS mock containers, each with a different mock host
 * 2) Mocking mYarnClient to return the mock hosts of the mock containers
 * 3) Mocking mRMClient.addContainerRequest to asynchronously call mMaster.onContainersAllocated
 * with a random container on a requested host
 *
 * @param numContainers the number of mock container hosts
 */
private void mockResourceManager(int numContainers) throws Exception {
    final Random random = new Random();
    final List<Container> mockContainers = Lists.newArrayList();
    List<NodeReport> nodeReports = Lists.newArrayList();
    List<String> hosts = Lists.newArrayList(MASTER_ADDRESS);
    for (int i = 0; i < numContainers - 1; i++) {
        String host = "host" + i;
        hosts.add(host);
    }
    for (String host : hosts) {
        Container mockContainer = Mockito.mock(Container.class);
        Mockito.when(mockContainer.getNodeHttpAddress()).thenReturn(host + ":8042");
        Mockito.when(mockContainer.getNodeId()).thenReturn(NodeId.newInstance(host, 0));
        mockContainers.add(mockContainer);
        NodeReport report = Mockito.mock(NodeReport.class);
        Mockito.when(report.getNodeId()).thenReturn(NodeId.newInstance(host, 0));
        nodeReports.add(report);
    }
    Mockito.when(YarnUtils.getNodeHosts(mYarnClient)).thenReturn(Sets.newHashSet(hosts));

    // Pretend to be the Resource Manager, allocating containers when they are requested.
    Mockito.doAnswer(new Answer<Void>() {
        @Override
        public Void answer(final InvocationOnMock invocation) {
            new Thread(new Runnable() {
                @Override
                public void run() {
                    // Allow the requests to interleave randomly
                    CommonUtils.sleepMs(50 + random.nextInt(200));
                    // Allocate a randomly chosen container from among the requested hosts
                    ContainerRequest request = invocation.getArgumentAt(0, ContainerRequest.class);
                    Set<String> requestedHosts = Sets.newHashSet(request.getNodes());
                    List<Container> requestedContainers = Lists.newArrayList();
                    for (Container container : mockContainers) {
                        if (requestedHosts.contains(container.getNodeId().getHost())) {
                            requestedContainers.add(container);
                        }
                    }
                    mMaster.onContainersAllocated(Lists
                            .newArrayList(requestedContainers.get(random.nextInt(requestedContainers.size()))));
                }
            }).start();
            return null;
        }
    }).when(mRMClient).addContainerRequest(Mockito.<ContainerRequest>any());
}

---

From source file:tachyon.yarn.ApplicationMasterTest.java

License:Apache License

/**
 * Tests that the Tachyon worker containers are launched properly.
 *///from w w  w  .  j  a va2 s .  c  om
@Test
public void launchTachyonWorkerContainersTest() throws Exception {
    Container mockContainer1 = Mockito.mock(Container.class);
    Container mockContainer2 = Mockito.mock(Container.class);
    // The containers must be from different hosts because we don't support multiple clients on the
    // same host.
    Mockito.when(mockContainer1.getNodeId()).thenReturn(NodeId.newInstance("host1", 0));
    Mockito.when(mockContainer2.getNodeId()).thenReturn(NodeId.newInstance("host2", 0));
    // Say that the master is allocated so that container offers are assumed to be worker offers
    mPrivateAccess.getMasterAllocated().countDown();
    mPrivateAccess.setMasterContainerAddress("masterAddress");
    mPrivateAccess.setOutstandingWorkerContainerRequestsLatch(new CountDownLatch(2));

    List<Container> containers = Lists.newArrayList(mockContainer1, mockContainer2);

    mMaster.onContainersAllocated(containers);
    Mockito.verify(mNMClient).startContainer(Mockito.same(mockContainer1),
            Mockito.argThat(getContextMatcher(EXPECTED_WORKER_CONTEXT)));
    Mockito.verify(mNMClient).startContainer(Mockito.same(mockContainer2),
            Mockito.argThat(getContextMatcher(EXPECTED_WORKER_CONTEXT)));
    Assert.assertEquals(containers.size(), mPrivateAccess.getWorkerHosts().size());
}

---

From source file:uk.ac.gla.terrier.probos.master.ProbosApplicationMasterDistributedServiceImpl.java

License:Open Source License

@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
    for (Container c : allocatedContainers) {
        allocatedNodes.add(c.getNodeId());
    }//  w  w  w  . j  a va  2s. c o m
    super.onContainersAllocated(allocatedContainers);
}

---

From source file:yarnkit.container.ContainerTracker.java

License:Apache License

public void launchContainer(@Nonnull Container container) {
    LOG.info("Launching a container '" + container.getId() + "' on node: " + container.getNodeId());

    Preconditions.checkNotNull(clcFactory, "ContainerLaunchContextFactory should not be null");

    needed.decrementAndGet();/*from  w  ww  . j a  va2s.  c  o  m*/
    runningContainers.put(container.getId(), container);
    nodeManager.startContainerAsync(container, clcFactory.duplicate(context));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »