Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:gobblin.yarn.YarnService.java

License:Apache License

@SuppressWarnings("unused")
@Subscribe/*from  w ww . j  a  va 2s  .c om*/
public void handleContainerShutdownRequest(ContainerShutdownRequest containerShutdownRequest) {
    for (Container container : containerShutdownRequest.getContainers()) {
        LOGGER.info(
                String.format("Stopping container %s running on %s", container.getId(), container.getNodeId()));
        this.nmClientAsync.stopContainerAsync(container.getId(), container.getNodeId());
    }
}

---

From source file:husky.server.ContainerRunnable.java

License:Apache License

public ContainerRunnable(HuskyApplicationMaster appMaster, HuskyRMCallbackHandler rmCallbackHandler,
        Container container) {
    LOG.info("New container " + container.getId() + " starts on " + container.getNodeId().getHost());
    mContainer = container;//from   www . j a va 2 s .co m
    mAppMaster = appMaster;
    mRMCallbackHandler = rmCallbackHandler;
}

---

From source file:io.hops.tensorflow.TimelineHandler.java

License:Apache License

public void publishContainerStartEvent(Container container) {
    final TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(container.getId().toString());
    entity.setEntityType(ApplicationMaster.YarntfEntity.YARNTF_CONTAINER.toString());
    entity.setDomainId(domainId);//from   ww w . j  av a 2  s .c o  m
    entity.addPrimaryFilter("user", ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(ApplicationMaster.YarntfEvent.YARNTF_CONTAINER_START.toString());
    event.addEventInfo("Node", container.getNodeId().toString());
    event.addEventInfo("Resources", container.getResource().toString());
    entity.addEvent(event);

    try {
        ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
            @Override
            public TimelinePutResponse run() throws Exception {
                return timelineClient.putEntities(entity);
            }
        });
    } catch (Exception e) {
        LOG.error("Container start event could not be published for " + container.getId().toString(),
                e instanceof UndeclaredThrowableException ? e.getCause() : e);
    }
}

---

From source file:io.hops.util.DBUtility.java

License:Apache License

public static void removeContainersToDecrease(
        final Collection<org.apache.hadoop.yarn.api.records.Container> containers) throws IOException {
    long start = System.currentTimeMillis();
    AsyncLightWeightRequestHandler removeContainerToDecrease = new AsyncLightWeightRequestHandler(
            YARNOperationType.TEST) {/*from   w w  w . jav  a 2s  .  c o  m*/
        @Override
        public Object performTask() throws StorageException {
            connector.beginTransaction();
            connector.writeLock();
            ContainerToDecreaseDataAccess ctsDA = (ContainerToDecreaseDataAccess) RMStorageFactory
                    .getDataAccess(ContainerToDecreaseDataAccess.class);
            List<io.hops.metadata.yarn.entity.Container> containersToDecrease = new ArrayList<io.hops.metadata.yarn.entity.Container>();
            for (org.apache.hadoop.yarn.api.records.Container container : containers) {
                containersToDecrease.add(new io.hops.metadata.yarn.entity.Container(
                        container.getId().toString(), container.getNodeId().toString(),
                        container.getNodeHttpAddress(), container.getPriority().getPriority(),
                        container.getResource().getMemorySize(), container.getResource().getVirtualCores(),
                        container.getResource().getGPUs(), container.getVersion()));
            }
            ctsDA.removeAll(containersToDecrease);
            connector.commit();
            return null;
        }
    };

    removeContainerToDecrease.handle();
    long duration = System.currentTimeMillis() - start;
    if (duration > 10) {
        LOG.error("too long " + duration);
    }
}

---

From source file:io.hops.util.DBUtility.java

License:Apache License

public static void addContainersToDecrease(final List<org.apache.hadoop.yarn.api.records.Container> containers)
        throws IOException {
    long start = System.currentTimeMillis();
    AsyncLightWeightRequestHandler addContainerToDecrease = new AsyncLightWeightRequestHandler(
            YARNOperationType.TEST) {//  w  w w  . j  a  va  2s  .c  om
        @Override
        public Object performTask() throws StorageException {
            connector.beginTransaction();
            connector.writeLock();
            ContainerToDecreaseDataAccess ctsDA = (ContainerToDecreaseDataAccess) RMStorageFactory
                    .getDataAccess(ContainerToDecreaseDataAccess.class);
            List<io.hops.metadata.yarn.entity.Container> containersToDecrease = new ArrayList<io.hops.metadata.yarn.entity.Container>();
            for (org.apache.hadoop.yarn.api.records.Container container : containers) {
                containersToDecrease.add(new io.hops.metadata.yarn.entity.Container(
                        container.getId().toString(), container.getNodeId().toString(),
                        container.getNodeHttpAddress(), container.getPriority().getPriority(),
                        container.getResource().getMemorySize(), container.getResource().getVirtualCores(),
                        container.getResource().getGPUs(), container.getVersion()));
            }
            ctsDA.addAll(containersToDecrease);
            connector.commit();
            return null;
        }
    };

    addContainerToDecrease.handle();
    long duration = System.currentTimeMillis() - start;
    if (duration > 10) {
        LOG.error("too long " + duration);
    }
}

---

From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java

License:Apache License

private void startTaskTimeoutExecutor() {
    this.taskTimeoutExecutor.submit(new Runnable() {
        @Override//from  w ww.  j a  v a  2  s  .  c om
        public void run() {
            while (true) {
                try {
                    Thread.sleep(GuaguaAppMaster.this.taskTimeOut);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    break;
                }
                LOG.debug(GuaguaAppMaster.this.partitionProgress.toString());
                for (Map.Entry<Integer, GuaguaIterationStatus> entry : GuaguaAppMaster.this.partitionProgress
                        .entrySet()) {
                    GuaguaIterationStatus status = entry.getValue();
                    // doesn't work in the first iteration
                    if (status.getTime() != 0l && status.getCurrentIteration() != 1
                            && (System.currentTimeMillis()
                                    - status.getTime()) > GuaguaAppMaster.this.taskTimeOut) {
                        List<Container> containers = GuaguaAppMaster.this.partitionContainerMap
                                .get(entry.getKey());
                        Container container = containers.get(containers.size() - 1);
                        LOG.info(
                                "Container {} is timeout with timeout period {}, will be killed by node manager {}.",
                                container.getId(), GuaguaAppMaster.this.taskTimeOut, container.getNodeId());
                        GuaguaAppMaster.this.getNmClientAsync().stopContainerAsync(container.getId(),
                                container.getNodeId());
                    }
                }
            }
        }
    });
}

---

From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java

License:Apache License

/**
 * For each container successfully allocated, attempt to set up and launch a Guagua worker/master task.
 * /*from  www .j  av a2  s.c om*/
 * @param allocatedContainers
 *            the containers we have currently allocated.
 */
private void startContainerLaunchingThreads(final List<Container> allocatedContainers) {
    Map<String, List<Container>> hostContainterMap = getHostContainersMap(allocatedContainers);
    int size = allocatedContainers.size();
    while (size > 0) {
        int currentPartition = getCurrentPartition();
        if (currentPartition == -1) {
            LOG.warn("Request too many resources. TODO, remove containers no needed.");
            for (Container container : allocatedContainers) {
                GuaguaAppMaster.this.getAmRMClient().releaseAssignedContainer(container.getId());
            }
            break;
        }
        Container container = getDataLocalityContainer(hostContainterMap, currentPartition);
        if (container == null) {
            container = allocatedContainers.get(0);
        }

        allocatedContainers.remove(container);

        LOG.info(
                "Launching command on a new container., containerId={}, containerNode={}, containerPort={}, containerNodeURI={}, containerResourceMemory={}",
                container.getId(), container.getNodeId().getHost(), container.getNodeId().getPort(),
                container.getNodeHttpAddress(), container.getResource().getMemory());

        List<Container> list = this.partitionContainerMap.get(currentPartition);
        if (list == null) {
            list = new ArrayList<Container>();
        }
        list.add(container);
        this.partitionContainerMap.put(currentPartition, list);
        this.containerPartitionMap.put(container.getId().toString(), currentPartition);
        this.partitionStatusMap.put(currentPartition, PartitionStatus.INIT);
        LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(container,
                getContainerListener(), currentPartition);
        getExecutor().execute(runnableLaunchContainer);

        size = allocatedContainers.size();
    }
}

---

From source file:ml.shifu.guagua.yarn.GuaguaAppMaster.java

License:Apache License

private Map<String, List<Container>> getHostContainersMap(final List<Container> allocatedContainers) {
    Map<String, List<Container>> hostContainterMap = new HashMap<String, List<Container>>();
    for (Container container : allocatedContainers) {
        String host = container.getNodeId().getHost();
        List<Container> containers = hostContainterMap.get(host);
        if (containers == null) {
            containers = new ArrayList<Container>();
        }/*w w w. j ava 2  s.  com*/
        containers.add(container);
        hostContainterMap.put(host, containers);
    }
    return hostContainterMap;
}

---

From source file:org.apache.drill.yarn.appMaster.AMYarnFacadeImpl.java

License:Apache License

@Override
public void killContainer(Container container) {
    nodeMgr.stopContainerAsync(container.getId(), container.getNodeId());
}

---

From source file:org.apache.drill.yarn.appMaster.ClusterControllerImpl.java

License:Apache License

@Override
public synchronized void containersAllocated(List<Container> containers) {
    EventContext context = new EventContext(this);
    for (Container container : containers) {
        if (allocatedContainers.contains(container.getId())) {
            continue;
        }// ww  w.j  a  v a2  s.  c o  m

        // We should never get a container on a node in the blacklist we
        // sent to YARN. If we do, something is wrong. Log the error and
        // reject the container. Else, bad things happen further along as
        // the tracking mechanisms assume one task per node.

        String host = container.getNodeId().getHost();
        if (nodeInventory.isInUse(host)) {
            LOG.error("Host is in use, but YARN allocated a container: " + DoYUtil.labelContainer(container)
                    + " - container rejected.");
            yarn.releaseContainer(container);
            continue;
        }

        // The container is fine.

        allocatedContainers.add(container.getId());
        int priority = container.getPriority().getPriority();
        int offset = priority - PRIORITY_OFFSET;
        if (offset < 0 || offset > prioritizedGroups.size()) {
            LOG.error("Container allocated with unknown priority " + DoYUtil.labelContainer(container));
            continue;
        }
        context.setGroup(prioritizedGroups.get(offset));
        context.group.containerAllocated(context, container);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »