Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

List of usage examples for org.apache.hadoop.yarn.api.records Container getNodeId

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. Container
  6. getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:org.apache.flink.yarn.YarnResourceManagerTest.java

License:Apache License

private static Container mockContainer(String host, int port, int containerId) {
    Container mockContainer = mock(Container.class);

    NodeId mockNodeId = NodeId.newInstance(host, port);
    ContainerId mockContainerId = ContainerId.newInstance(
            ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1),
            containerId);/* ww w .  j  a  va 2s.co  m*/

    when(mockContainer.getId()).thenReturn(mockContainerId);
    when(mockContainer.getNodeId()).thenReturn(mockNodeId);
    when(mockContainer.getResource()).thenReturn(Resource.newInstance(200, 1));
    when(mockContainer.getPriority()).thenReturn(Priority.UNDEFINED);

    return mockContainer;
}

From source file:org.apache.giraph.yarn.GiraphApplicationMaster.java

License:Apache License

/**
 * For each container successfully allocated, attempt to set up and launch
 * a Giraph worker/master task./*from w  w w .  jav  a2s.  com*/
 * @param allocatedContainers the containers we have currently allocated.
 */
private void startContainerLaunchingThreads(final List<Container> allocatedContainers) {
    for (Container allocatedContainer : allocatedContainers) {
        LOG.info("Launching command on a new container." + ", containerId=" + allocatedContainer.getId()
                + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":"
                + allocatedContainer.getNodeId().getPort() + ", containerNodeURI="
                + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory="
                + allocatedContainer.getResource().getMemory());
        // Launch and start the container on a separate thread to keep the main
        // thread unblocked as all containers may not be allocated at one go.
        LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(allocatedContainer,
                containerListener);
        executor.execute(runnableLaunchContainer);
    }
}

From source file:org.apache.gobblin.yarn.YarnService.java

License:Apache License

/**
 * Request the Resource Manager to release the container
 * @param containerReleaseRequest containers to release
 *///w  w  w.  j  a  v  a2 s  .  co m
@Subscribe
public void handleContainerReleaseRequest(ContainerReleaseRequest containerReleaseRequest) {
    for (Container container : containerReleaseRequest.getContainers()) {
        LOGGER.info(String.format("Releasing container %s running on %s", container.getId(),
                container.getNodeId()));

        // Record that this container was explicitly released so that a new one is not spawned to replace it
        // Put the container id in the releasedContainerCache before releasing it so that handleContainerCompletion()
        // can check for the container id and skip spawning a replacement container.
        // Note that this is best effort since these are asynchronous operations and a container may abort concurrently
        // with the release call. So in some cases a replacement container may have already been spawned before
        // the container is put into the black list.
        this.releasedContainerCache.put(container.getId(), "");
        this.amrmClientAsync.releaseAssignedContainer(container.getId());
    }
}

From source file:org.apache.hama.bsp.JobImpl.java

License:Apache License

@Override
public JobState startJob() throws Exception {

    this.allocatedContainers = new ArrayList<Container>(numBSPTasks);
    NMTokenCache nmTokenCache = new NMTokenCache();
    while (allocatedContainers.size() < numBSPTasks) {
        AllocateRequest req = AllocateRequest.newInstance(lastResponseID, 0.0f,
                createBSPTaskRequest(numBSPTasks - allocatedContainers.size(), taskMemoryInMb, priority),
                releasedContainers, null);

        AllocateResponse allocateResponse = resourceManager.allocate(req);
        for (NMToken token : allocateResponse.getNMTokens()) {
            nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
        }//w w  w .  ja va 2s.c  o  m

        LOG.info("Got response ID: " + allocateResponse.getResponseId() + " with num of containers: "
                + allocateResponse.getAllocatedContainers().size() + " and following resources: "
                + allocateResponse.getAvailableResources().getMemory() + "mb");
        this.lastResponseID = allocateResponse.getResponseId();

        this.allocatedContainers.addAll(allocateResponse.getAllocatedContainers());

        LOG.info("Waiting to allocate " + (numBSPTasks - allocatedContainers.size()) + " more containers...");

        Thread.sleep(1000l);
    }

    LOG.info("Got " + allocatedContainers.size() + " containers!");

    int id = 0;
    for (Container allocatedContainer : allocatedContainers) {
        LOG.info("Launching task on a new container." + ", containerId=" + allocatedContainer.getId()
                + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":"
                + allocatedContainer.getNodeId().getPort() + ", containerNodeURI="
                + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory"
                + allocatedContainer.getResource().getMemory());

        // Connect to ContainerManager on the allocated container
        String user = conf.get("bsp.user.name");
        if (user == null) {
            user = System.getenv(ApplicationConstants.Environment.USER.name());
        }

        ContainerManagementProtocol cm = null;
        try {
            cm = getContainerManagementProtocolProxy(yarnRPC,
                    nmTokenCache.getToken(allocatedContainer.getNodeId().toString()),
                    allocatedContainer.getNodeId(), user);
        } catch (Exception e) {
            LOG.error("Failed to create ContainerManager...");
            if (cm != null)
                yarnRPC.stopProxy(cm, conf);
            e.printStackTrace();
        }

        BSPTaskLauncher runnableLaunchContainer = new BSPTaskLauncher(id, allocatedContainer, cm, conf, jobFile,
                jobId);

        launchers.put(id, runnableLaunchContainer);
        runnableLaunchContainer.start();
        completionQueue.add(runnableLaunchContainer);
        id++;
    }

    LOG.info("Waiting for tasks to finish...");
    state = JobState.RUNNING;
    int completed = 0;

    List<Integer> cleanupTasks = new ArrayList<Integer>();
    while (completed != numBSPTasks) {
        for (BSPTaskLauncher task : completionQueue) {
            BSPTaskStatus returnedTask = task.poll();
            // if our task returned with a finished state
            if (returnedTask != null) {
                if (returnedTask.getExitStatus() != 0) {
                    LOG.error("Task with id \"" + returnedTask.getId() + "\" failed!");
                    cleanupTask(returnedTask.getId());
                    state = JobState.FAILED;
                    return state;
                } else {
                    LOG.info("Task \"" + returnedTask.getId() + "\" sucessfully finished!");
                    completed++;
                    LOG.info("Waiting for " + (numBSPTasks - completed) + " tasks to finish!");
                }
                cleanupTasks.add(returnedTask.getId());
            }
        }
        Thread.sleep(1000L);
    }

    for (Integer stopId : cleanupTasks) {
        cleanupTask(stopId);
    }

    state = JobState.SUCCESS;
    return state;
}

From source file:org.apache.helix.provisioning.yarn.GenericApplicationMaster.java

License:Apache License

public ListenableFuture<ContainerStopResponse> stopContainer(Container container) {
    LOG.info("Requesting container STOP:" + container);
    SettableFuture<ContainerStopResponse> future = SettableFuture.create();
    containerStopMap.put(container.getId(), future);
    nmClientAsync.stopContainerAsync(container.getId(), container.getNodeId());
    return future;
}

From source file:org.apache.helix.provisioning.yarn.NMCallbackHandler.java

License:Apache License

@Override
public void onContainerStopped(ContainerId containerId) {
    LOG.info("Succeeded to stop Container " + containerId);
    Container container = containers.get(containerId);
    if (container != null) {
        applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId());
    }//from   w w  w  .java2s . com
    SettableFuture<ContainerStopResponse> settableFuture = applicationMaster.containerStopMap
            .remove(containerId);
    ContainerStopResponse value = new ContainerStopResponse();
    settableFuture.set(value);
    containers.remove(containerId);
}

From source file:org.apache.helix.provisioning.yarn.NMCallbackHandler.java

License:Apache License

@Override
public void onContainerStarted(ContainerId containerId, Map<String, ByteBuffer> allServiceResponse) {
    LOG.debug("Succeeded to start Container " + containerId);

    Container container = containers.get(containerId);
    if (container != null) {
        applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId());
    }/*  w ww. j ava2s  . c om*/
    SettableFuture<ContainerLaunchResponse> settableFuture = applicationMaster.containerLaunchResponseMap
            .remove(containerId);
    ContainerLaunchResponse value = new ContainerLaunchResponse();
    settableFuture.set(value);
}

From source file:org.apache.helix.provisioning.yarn.RMCallbackHandler.java

License:Apache License

@Override
public void onContainersAllocated(List<Container> allocatedContainers) {
    GenericApplicationMaster.LOG/*  w  w  w  . j ava  2  s  .c  o m*/
            .info("Got response from RM for container ask, allocatedCnt=" + allocatedContainers.size());
    for (Container allocatedContainer : allocatedContainers) {
        GenericApplicationMaster.LOG.info("Allocated new container." + ", containerId="
                + allocatedContainer.getId() + ", containerNode=" + allocatedContainer.getNodeId().getHost()
                + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI="
                + allocatedContainer.getNodeHttpAddress() + ", containerResourceMemory"
                + allocatedContainer.getResource().getMemory());
        for (ContainerRequest containerRequest : _genericApplicationMaster.containerRequestMap.keySet()) {
            if (containerRequest.getCapability().getMemory() == allocatedContainer.getResource().getMemory()) {
                SettableFuture<ContainerAskResponse> future = _genericApplicationMaster.containerRequestMap
                        .remove(containerRequest);
                ContainerAskResponse response = new ContainerAskResponse();
                response.setContainer(allocatedContainer);
                _genericApplicationMaster.allocatedContainerSet.add(allocatedContainer.getId());
                future.set(response);
                break;
            }
        }
    }
}

From source file:org.apache.hoya.tools.HoyaUtils.java

License:Apache License

public static String containerToString(Container container) {
    if (container == null) {
        return "null container";
    }//from  ww w  . j a  v  a 2  s  . co  m
    return String.format(Locale.ENGLISH, "ContainerID=%s nodeID=%s http=%s priority=%s", container.getId(),
            container.getNodeId(), container.getNodeHttpAddress(), container.getPriority());
}

From source file:org.apache.hoya.yarn.appmaster.state.AppState.java

License:Apache License

/**
 * handle completed node in the CD -move something from the live
 * server list to the completed server list
 * @param amConf YarnConfiguration/* w  ww .  j  a va  2 s . c o  m*/
 * @param status the node that has just completed
 * @return NodeCompletionResult
 */
public synchronized NodeCompletionResult onCompletedNode(YarnConfiguration amConf, ContainerStatus status) {
    ContainerId containerId = status.getContainerId();
    NodeCompletionResult result = new NodeCompletionResult();
    RoleInstance roleInstance;

    if (containersBeingReleased.containsKey(containerId)) {
        log.info("Container was queued for release");
        Container container = containersBeingReleased.remove(containerId);
        RoleStatus roleStatus = lookupRoleStatus(container);
        log.info("decrementing role count for role {}", roleStatus.getName());
        roleStatus.decReleasing();
        roleStatus.decActual();
        roleStatus.incCompleted();
        roleHistory.onReleaseCompleted(container);

    } else if (surplusNodes.remove(containerId)) {
        //its a surplus one being purged
        result.surplusNode = true;
    } else {
        //a container has failed 
        result.containerFailed = true;
        roleInstance = activeContainers.remove(containerId);
        if (roleInstance != null) {
            //it was active, move it to failed 
            incFailedCountainerCount();
            failedNodes.put(containerId, roleInstance);
        } else {
            // the container may have been noted as failed already, so look
            // it up
            roleInstance = failedNodes.get(containerId);
        }
        if (roleInstance != null) {
            int roleId = roleInstance.roleId;
            log.info("Failed container in role {}", roleId);
            try {
                RoleStatus roleStatus = lookupRoleStatus(roleId);
                roleStatus.decActual();
                boolean shortLived = isShortLived(roleInstance);
                String message;
                if (roleInstance.container != null) {
                    String user = null;
                    try {
                        user = HoyaUtils.getCurrentUser().getShortUserName();
                    } catch (IOException ioe) {
                    }
                    String completedLogsUrl = null;
                    Container c = roleInstance.container;
                    String url = null;
                    if (amConf != null) {
                        url = amConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
                    }
                    if (user != null && url != null) {
                        completedLogsUrl = url + "/" + c.getNodeId() + "/" + roleInstance.getContainerId()
                                + "/ctx/" + user;
                    }
                    message = String.format(
                            "Failure %s on host %s" + (completedLogsUrl != null ? ", see %s" : ""),
                            roleInstance.getContainerId(), c.getNodeId().getHost(), completedLogsUrl);
                } else {
                    message = String.format("Failure %s", containerId.toString());
                }
                roleStatus.noteFailed(message);
                //have a look to see if it short lived
                if (shortLived) {
                    roleStatus.incStartFailed();
                }

                if (roleInstance.container != null) {
                    roleHistory.onFailedContainer(roleInstance.container, shortLived);
                }

            } catch (YarnRuntimeException e1) {
                log.error("Failed container of unknown role {}", roleId);
            }
        } else {
            //this isn't a known container.

            log.error("Notified of completed container {} that is not in the list"
                    + " of active or failed containers", containerId);
            completionOfUnknownContainerEvent.incrementAndGet();
        }
    }

    if (result.surplusNode) {
        //a surplus node
        return result;
    }

    //record the complete node's details; this pulls it from the livenode set 
    //remove the node
    ContainerId id = status.getContainerId();
    RoleInstance node = getLiveNodes().remove(id);
    if (node == null) {
        log.warn("Received notification of completion of unknown node {}", id);
        completionOfNodeNotInLiveListEvent.incrementAndGet();

    } else {
        node.state = ClusterDescription.STATE_DESTROYED;
        node.exitCode = status.getExitStatus();
        node.diagnostics = status.getDiagnostics();
        getCompletedNodes().put(id, node);
        result.roleInstance = node;
    }
    return result;
}