Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:org.apache.hoya.yarn.appmaster.state.AppState.java

License:Apache License

/**
 * Event handler for allocated containers: builds up the lists
 * of assignment actions (what to run where), and possibly
 * a list of release operations/*  www  . j  a va 2 s  .  co m*/
 * @param allocatedContainers the containers allocated
 * @param assignments the assignments of roles to containers
 * @param releaseOperations any release operations
 */
public synchronized void onContainersAllocated(List<Container> allocatedContainers,
        List<ContainerAssignment> assignments, List<AbstractRMOperation> releaseOperations) {
    assignments.clear();
    releaseOperations.clear();
    List<Container> ordered = roleHistory.prepareAllocationList(allocatedContainers);
    for (Container container : ordered) {
        String containerHostInfo = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
        int allocated;
        int desired;
        //get the role
        ContainerId cid = container.getId();
        RoleStatus role = lookupRoleStatus(container);

        //dec requested count
        role.decRequested();
        //inc allocated count -this may need to be dropped in a moment,
        // but us needed to update the logic below
        allocated = role.incActual();

        //look for (race condition) where we get more back than we asked
        desired = role.getDesired();

        roleHistory.onContainerAllocated(container, desired, allocated);

        if (allocated > desired) {
            log.info("Discarding surplus container {} on {}", cid, containerHostInfo);
            releaseOperations.add(new ContainerReleaseOperation(cid));
            //register as a surplus node
            surplusNodes.add(cid);
            surplusContainers.incrementAndGet();
            //and, as we aren't binding it to role, dec that role's actual count
            role.decActual();
        } else {

            String roleName = role.getName();
            log.info("Assigning role {} to container" + " {}," + " on {}:{},", roleName, cid,
                    container.getNodeId().getHost(), container.getNodeId().getPort());

            assignments.add(new ContainerAssignment(container, role));
            //add to the history
            roleHistory.onContainerAssigned(container);
        }
    }
}

---

From source file:org.apache.hoya.yarn.appmaster.state.AppState.java

License:Apache License

/**
 * Add a restarted container by walking it through the create/submit/start
 * lifecycle, so building up the internal structures
 * @param container container that was running before the AM restarted
 * @throws HoyaRuntimeException on problems
 *///from  w  w w  . j a  va2  s.c  o m
private void addRestartedContainer(Container container) throws BadClusterStateException {
    String containerHostInfo = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
    // get the container ID
    ContainerId cid = container.getId();

    // get the role
    int roleId = ContainerPriority.extractRole(container);
    RoleStatus role = lookupRoleStatus(roleId);
    // increment its count
    role.incActual();
    String roleName = role.getName();

    log.info("Rebuilding container {} in role {} on {},", cid, roleName, containerHostInfo);

    //update app state internal structures and maps

    RoleInstance instance = new RoleInstance(container);
    instance.command = roleName;
    instance.role = roleName;
    instance.roleId = roleId;
    instance.environment = new String[0];
    instance.container = container;
    instance.createTime = now();
    instance.state = ClusterDescription.STATE_LIVE;
    activeContainers.put(cid, instance);
    //role history gets told
    roleHistory.onContainerAssigned(container);
    // pretend the container has just had its start actions submitted
    containerStartSubmitted(container, instance);
    // now pretend it has just started
    innerOnNodeManagerContainerStarted(cid);
}

---

From source file:org.apache.hoya.yarn.appmaster.state.RoleHistoryUtils.java

License:Apache License

public static String hostnameOf(Container container) {
    NodeId nodeId = container.getNodeId();
    if (nodeId == null) {
        throw new HoyaRuntimeException("Container has no node ID: %s", HoyaUtils.containerToString(container));
    }//from  w w  w  .  j a v a 2  s  . co m
    return nodeId.getHost();
}

---

From source file:org.apache.hoya.yarn.appmaster.state.RoleInstance.java

License:Apache License

public RoleInstance(Container container) {
    this.container = container;
    if (container == null) {
        throw new NullPointerException("Null container");
    }//from   w w  w . j a  va 2 s.c o  m
    if (container.getId() == null) {
        throw new NullPointerException("Null container ID");
    }
    id = container.getId().toString();
    if (container.getNodeId() != null) {
        host = container.getNodeId().getHost();
    }
    if (container.getNodeHttpAddress() != null) {
        hostURL = "http://" + container.getNodeHttpAddress();
    }
}

---

From source file:org.apache.ignite.yarn.ApplicationMaster.java

License:Apache License

/** {@inheritDoc} */
public synchronized void onContainersAllocated(List<Container> conts) {
    for (Container c : conts) {
        if (checkContainer(c)) {
            try {
                ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

                if (UserGroupInformation.isSecurityEnabled())
                    // Set the tokens to the newly allocated container:
                    ctx.setTokens(allTokens.duplicate());

                Map<String, String> env = new HashMap<>(System.getenv());

                env.put("IGNITE_TCP_DISCOVERY_ADDRESSES", getAddress(c.getNodeId().getHost()));

                if (props.jvmOpts() != null && !props.jvmOpts().isEmpty())
                    env.put("JVM_OPTS", props.jvmOpts());

                ctx.setEnvironment(env);

                Map<String, LocalResource> resources = new HashMap<>();

                resources.put("ignite", IgniteYarnUtils.setupFile(ignitePath, fs, LocalResourceType.ARCHIVE));
                resources.put("ignite-config.xml",
                        IgniteYarnUtils.setupFile(cfgPath, fs, LocalResourceType.FILE));

                if (props.licencePath() != null)
                    resources.put("gridgain-license.xml", IgniteYarnUtils
                            .setupFile(new Path(props.licencePath()), fs, LocalResourceType.FILE));

                if (props.userLibs() != null)
                    resources.put("libs",
                            IgniteYarnUtils.setupFile(new Path(props.userLibs()), fs, LocalResourceType.FILE));

                ctx.setLocalResources(resources);

                ctx.setCommands(Collections.singletonList(
                        (props.licencePath() != null ? "cp gridgain-license.xml ./ignite/*/ || true && " : "")
                                + "cp -r ./libs/* ./ignite/*/libs/ || true && " + "./ignite/*/bin/ignite.sh "
                                + "./ignite-config.xml" + " -J-Xmx" + ((int) props.memoryPerNode()) + "m"
                                + " -J-Xms" + ((int) props.memoryPerNode()) + "m"
                                + IgniteYarnUtils.YARN_LOG_OUT));

                log.log(Level.INFO, "Launching container: {0}.", c.getId());

                nmClient.startContainer(c, ctx);

                containers.put(c.getId(), new IgniteContainer(c.getId(), c.getNodeId(),
                        c.getResource().getVirtualCores(), c.getResource().getMemory()));
            } catch (Exception ex) {
                log.log(Level.WARNING, "Error launching container " + c.getId(), ex);
            }//w  ww.j  a v  a 2  s .  c o  m
        } else
            rmClient.releaseAssignedContainer(c.getId());
    }
}

---

From source file:org.apache.ignite.yarn.ApplicationMaster.java

License:Apache License

/**
 * @param cont Container.//from w  w  w. j  av  a  2s. c o  m
 * @return {@code True} if container satisfies requirements.
 */
private boolean checkContainer(Container cont) {
    // Check limit on running nodes.
    if (props.instances() <= containers.size())
        return false;

    // Check host name
    if (props.hostnameConstraint() != null
            && props.hostnameConstraint().matcher(cont.getNodeId().getHost()).matches())
        return false;

    // Check that slave satisfies min requirements.
    if (cont.getResource().getVirtualCores() < props.cpusPerNode()
            || cont.getResource().getMemory() < props.totalMemoryPerNode()) {
        log.log(Level.FINE, "Container resources not sufficient requirements. Host: {0}, cpu: {1}, mem: {2}",
                new Object[] { cont.getNodeId().getHost(), cont.getResource().getVirtualCores(),
                        cont.getResource().getMemory() });

        return false;
    }

    return true;
}

---

From source file:org.apache.metron.maas.service.callback.ContainerRequestListener.java

License:Apache License

public void removeContainers(int number, ModelRequest request) {
    int i = 0;//  w  ww.java 2  s. c o  m
    for (Container c : state.getList(request)) {
        if (i < number) {
            amRMClient.releaseAssignedContainer(c.getId());
            nmClient.stopContainerAsync(c.getId(), c.getNodeId());
        } else {
            break;
        }
        i++;
    }
}

---

From source file:org.apache.metron.maas.service.callback.ContainerRequestListener.java

License:Apache License

@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
    LOG.info("Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size());
    for (Container allocatedContainer : allocatedContainers) {
        containers.put(allocatedContainer.getId(), allocatedContainer);
        state.registerContainer(allocatedContainer.getResource(), allocatedContainer);
        LOG.info("Launching shell command on a new container." + ", containerId=" + allocatedContainer.getId()
                + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":"
                + allocatedContainer.getNodeId().getPort() + ", containerNodeURI="
                + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory="
                + allocatedContainer.getResource().getMemory() + ", containerResourceVirtualCores="
                + allocatedContainer.getResource().getVirtualCores());
    }// www  .j a  va  2  s. c  o  m
}

---

From source file:org.apache.metron.maas.service.callback.ContainerRequestListener.java

License:Apache License

@Override
public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Succeeded to start Container " + containerId);
    }/*from ww  w. ja  va2  s . c om*/
    Container container = containers.get(containerId);
    if (container != null) {
        nmClient.getContainerStatusAsync(containerId, container.getNodeId());
    }
    if (timelineClient != null && container != null) {
        YarnUtils.INSTANCE.publishContainerStartEvent(timelineClient, container, domainId, appSubmitterUgi);
    }
}

---

From source file:org.apache.metron.maas.service.yarn.YarnUtils.java

License:Apache License

public void publishContainerStartEvent(final TimelineClient timelineClient, Container container,
        String domainId, UserGroupInformation ugi) {
    final TimelineEntity entity = new TimelineEntity();
    entity.setEntityId("" + container.getId());
    entity.setEntityType(ApplicationMaster.DSEntity.DS_CONTAINER.toString());
    entity.setDomainId(domainId);//from ww  w.  j  a va 2 s  .c  o m
    entity.addPrimaryFilter("user", ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(ContainerEvents.CONTAINER_START.toString());
    event.addEventInfo("Node", container.getNodeId().toString());
    event.addEventInfo("Resources", container.getResource().toString());
    entity.addEvent(event);

    try {
        ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
            @Override
            public TimelinePutResponse run() throws Exception {
                return timelineClient.putEntities(entity);
            }
        });
    } catch (Exception e) {
        LOG.error("Container start event could not be published for " + container.getId().toString(),
                e instanceof UndeclaredThrowableException ? e.getCause() : e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »