Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

List of usage examples for org.apache.hadoop.yarn.api.records Container getNodeId

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. Container
  6. getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:org.apache.reef.runtime.yarn.driver.DefaultRackNameFormatter.java

License:Apache License

/**
 * @see RackNameFormatter#getRackName(Container)
 *//*from ww w .java  2 s . co  m*/
@Override
public String getRackName(final Container container) {
    final String hostName = container.getNodeId().getHost();

    // the rack name comes as part of the host name, e.g.
    // <rackName>-<hostNumber>
    // we perform some checks just in case it doesn't
    String rackName = null;
    if (hostName != null) {
        final String[] rackNameAndNumber = hostName.split("-");
        if (rackNameAndNumber.length == 2) {
            rackName = rackNameAndNumber[0];
        } else {
            LOG.log(Level.WARNING, "Could not get information from the rack name, should use the default");
        }
    }

    return rackName;
}

From source file:org.apache.reef.runtime.yarn.driver.YarnContainerManager.java

License:Apache License

/**
 * Handles new container allocations. Calls come from YARN.
 *
 * @param container newly allocated/* w  ww.  ja  v  a2s . c om*/
 */
private void handleNewContainer(final Container container, final boolean isRecoveredContainer) {

    LOG.log(Level.FINE, "allocated container: id[ {0} ]", container.getId());
    // recovered container is not new allocation, it is just checking back from previous driver failover
    if (!isRecoveredContainer) {
        synchronized (this) {
            if (matchContainerWithPendingRequest(container)) {
                final AMRMClient.ContainerRequest matchedRequest = this.requestsAfterSentToRM.peek();
                this.containerRequestCounter.decrement();
                this.containers.add(container);

                LOG.log(Level.FINEST, "{0} matched with {1}",
                        new Object[] { container.toString(), matchedRequest.toString() });

                // Due to the bug YARN-314 and the workings of AMRMCClientAsync, when x-priority m-capacity zero-container request
                // and x-priority n-capacity nonzero-container request are sent together, where m > n, RM ignores the latter.
                // Therefore it is necessary avoid sending zero-container request, even it means getting extra containers.
                // It is okay to send nonzero m-capacity and n-capacity request together since bigger containers can be matched.
                // TODO: revisit this when implementing locality-strictness (i.e. a specific rack request can be ignored)
                if (this.requestsAfterSentToRM.size() > 1) {
                    try {
                        this.resourceManager.removeContainerRequest(matchedRequest);
                    } catch (final Exception e) {
                        LOG.log(Level.WARNING,
                                "Nothing to remove from Async AMRM client's queue, removal attempt failed with exception",
                                e);
                    }
                }

                this.requestsAfterSentToRM.remove();
                doHomogeneousRequests();

                LOG.log(Level.FINEST, "Allocated Container: memory = {0}, core number = {1}", new Object[] {
                        container.getResource().getMemory(), container.getResource().getVirtualCores() });
                this.reefEventHandlers.onResourceAllocation(ResourceAllocationProto.newBuilder()
                        .setIdentifier(container.getId().toString()).setNodeId(container.getNodeId().toString())
                        .setResourceMemory(container.getResource().getMemory())
                        .setVirtualCores(container.getResource().getVirtualCores()).build());
                // we only add this to Container log after the Container has been registered as an REEF Evaluator.
                logContainerAddition(container.getId().toString());
                this.updateRuntimeStatus();
            } else {
                LOG.log(Level.WARNING, "Got an extra container {0} that doesn't match, releasing...",
                        container.getId());
                this.resourceManager.releaseAssignedContainer(container.getId());
            }
        }
    }
}

From source file:org.apache.reef.runtime.yarn.driver.YarnContainerManager.java

License:Apache License

/**
 * Match to see whether the container satisfies the request.
 * We take into consideration that RM has some freedom in rounding
 * up the allocation and in placing containers on other machines.
 *///from   ww w .j  a  v  a 2s . c o  m
private boolean matchContainerWithPendingRequest(Container container) {
    if (this.requestsAfterSentToRM.isEmpty()) {
        return false;
    }

    final AMRMClient.ContainerRequest request = this.requestsAfterSentToRM.peek();
    final boolean resourceCondition = container.getResource().getMemory() >= request.getCapability()
            .getMemory(); // TODO: check vcores once YARN-2380 is resolved
    final boolean nodeCondition = request.getNodes() == null
            || request.getNodes().contains(container.getNodeId().getHost());
    final boolean rackCondition = request.getRacks() == null
            || request.getRacks().contains(this.nodeIdToRackName.get(container.getNodeId().toString()));

    return resourceCondition && (request.getRelaxLocality() || (rackCondition && nodeCondition));
}

From source file:org.apache.reef.runtime.yarn.driver.YarnDriverRuntimeRestartManager.java

License:Apache License

/**
 * Used by {@link org.apache.reef.driver.restart.DriverRestartManager}.
 * Gets the list of previous containers from the resource manager,
 * compares that list to the YarnDriverRuntimeRestartManager's own list based on the evalutor preserver,
 * and determine which evaluators are alive and which have failed during restart.
 * @return a map of Evaluator ID to {@link EvaluatorRestartInfo} for evaluators that have either failed or survived
 * driver restart./*from w  ww . j  a va 2  s .c o  m*/
 */
@Override
public RestartEvaluators getPreviousEvaluators() {
    final RestartEvaluators.Builder restartEvaluatorsBuilder = RestartEvaluators.newBuilder();

    this.initializeListOfPreviousContainers();

    if (this.previousContainers != null && !this.previousContainers.isEmpty()) {
        LOG.log(Level.INFO, "Driver restarted, with {0} previous containers", this.previousContainers.size());
        final Set<String> expectedContainers = this.evaluatorPreserver.recoverEvaluators();

        final int numExpectedContainers = expectedContainers.size();
        final int numPreviousContainers = this.previousContainers.size();
        if (numExpectedContainers > numPreviousContainers) {
            // we expected more containers to be alive, some containers must have died during driver restart
            LOG.log(Level.WARNING, "Expected {0} containers while only {1} are still alive",
                    new Object[] { numExpectedContainers, numPreviousContainers });
            final Set<String> previousContainersIds = new HashSet<>();
            for (final Container container : this.previousContainers) {
                previousContainersIds.add(container.getId().toString());
            }
            for (final String expectedContainerId : expectedContainers) {
                if (!previousContainersIds.contains(expectedContainerId)) {
                    LOG.log(Level.WARNING,
                            "Expected container [{0}] not alive, must have failed during driver restart.",
                            expectedContainerId);
                    restartEvaluatorsBuilder.addRestartEvaluator(
                            EvaluatorRestartInfo.createFailedEvaluatorInfo(expectedContainerId));
                }
            }
        }
        if (numExpectedContainers < numPreviousContainers) {
            // somehow we have more alive evaluators, this should not happen
            throw new RuntimeException("Expected only [" + numExpectedContainers + "] containers "
                    + "but resource manager believe that [" + numPreviousContainers
                    + "] are outstanding for driver.");
        }

        //  numExpectedContainers == numPreviousContainers
        for (final Container container : this.previousContainers) {
            LOG.log(Level.FINE, "Previous container: [{0}]", container.toString());
            if (!expectedContainers.contains(container.getId().toString())) {
                throw new RuntimeException("Not expecting container " + container.getId().toString());
            }

            restartEvaluatorsBuilder.addRestartEvaluator(EvaluatorRestartInfo.createExpectedEvaluatorInfo(
                    ResourceEventImpl.newRecoveryBuilder().setIdentifier(container.getId().toString())
                            .setNodeId(container.getNodeId().toString())
                            .setRackName(rackNameFormatter.getRackName(container))
                            .setResourceMemory(container.getResource().getMemory())
                            .setVirtualCores(container.getResource().getVirtualCores()).build()));
        }
    }

    return restartEvaluatorsBuilder.build();
}

From source file:org.apache.samza.job.yarn.ContainerUtil.java

License:Apache License

public void runContainer(int samzaContainerId, Container container) {
    String containerIdStr = ConverterUtils.toString(container.getId());
    log.info("Got available container ID ({}) for container: {}", samzaContainerId, container);

    String cmdBuilderClassName;//from w w w.  java 2 s.c  o m
    if (taskConfig.getCommandClass().isDefined()) {
        cmdBuilderClassName = taskConfig.getCommandClass().get();
    } else {
        cmdBuilderClassName = ShellCommandBuilder.class.getName();
    }
    CommandBuilder cmdBuilder = (CommandBuilder) Util.getObj(cmdBuilderClassName);
    cmdBuilder.setConfig(config).setId(samzaContainerId).setUrl(state.coordinatorUrl);

    String command = cmdBuilder.buildCommand();
    log.info("Container ID {} using command {}", samzaContainerId, command);

    log.info("Container ID {} using environment variables: ", samzaContainerId);
    Map<String, String> env = new HashMap<String, String>();
    for (Map.Entry<String, String> entry : cmdBuilder.buildEnvironment().entrySet()) {
        String escapedValue = Util.envVarEscape(entry.getValue());
        env.put(entry.getKey(), escapedValue);
        log.info("{}={} ", entry.getKey(), escapedValue);
    }

    Path path = new Path(yarnConfig.getPackagePath());
    log.info("Starting container ID {} using package path {}", samzaContainerId, path);

    startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
            command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR));

    if (state.neededContainers.decrementAndGet() == 0) {
        state.jobHealthy.set(true);
    }
    state.runningContainers.put(samzaContainerId, new YarnContainer(container));

    log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).",
            new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(),
                    container.getNodeHttpAddress(), containerIdStr });

    log.info("Started container ID {}", samzaContainerId);
}

From source file:org.apache.samza.job.yarn.refactor.YarnClusterResourceManager.java

License:Apache License

/**
 * Callback invoked from Yarn when containers are allocated. This translates the yarn callbacks into Samza
 * specific ones.//from   ww w .jav  a 2s.  c o m
 * @param containers the list of {@link Container} returned by Yarn.
 */
@Override
public void onContainersAllocated(List<Container> containers) {
    List<SamzaResource> resources = new ArrayList<SamzaResource>();
    for (Container container : containers) {
        log.info("Container allocated from RM on " + container.getNodeId().getHost());
        final String id = container.getId().toString();
        String host = container.getNodeId().getHost();
        int memory = container.getResource().getMemory();
        int numCores = container.getResource().getVirtualCores();

        SamzaResource resource = new SamzaResource(numCores, memory, host, id);
        allocatedResources.put(resource, container);
        resources.add(resource);
    }
    _callback.onResourcesAvailable(resources);
}

From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java

License:Apache License

/**
 * Runs a process as specified by the command builder on the container.
 * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process)
 * @param container the samza container to run.
 * @param cmdBuilder the command builder that encapsulates the command, and the context
 *
 * @throws SamzaContainerLaunchException  when there's an exception in submitting the request to the RM.
 *
 *///from  ww  w  . j  a  va 2 s. c om
//TODO: we don't need samzaContainerId as a param here.
public void runContainer(int samzaContainerId, Container container, CommandBuilder cmdBuilder)
        throws SamzaContainerLaunchException {
    String containerIdStr = ConverterUtils.toString(container.getId());
    log.info("Got available container ID ({}) for container: {}", samzaContainerId, container);

    // check if we have framework path specified. If yes - use it, if not use default ./__package/
    String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries
    String cmdPath = "./__package/";

    String fwkPath = JobConfig.getFwkPath(config);
    if (fwkPath != null && (!fwkPath.isEmpty())) {
        cmdPath = fwkPath;
        jobLib = "export JOB_LIB_DIR=./__package/lib";
    }
    log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib);
    cmdBuilder.setCommandPath(cmdPath);

    String command = cmdBuilder.buildCommand();
    log.info("Container ID {} using command {}", samzaContainerId, command);

    Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
    printContainerEnvironmentVariables(samzaContainerId, env);

    log.info("Samza FWK path: " + command + "; env=" + env);

    Path path = new Path(yarnConfig.getPackagePath());
    log.info("Starting container ID {} using package path {}", samzaContainerId, path);

    startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib,
            command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR));

    log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).",
            new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(),
                    container.getNodeHttpAddress(), containerIdStr });

    log.info("Started container ID {}", samzaContainerId);
}

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

/**
 * Callback invoked from Yarn when containers are allocated. This translates the yarn callbacks into Samza
 * specific ones./*from  www.j  a v a  2  s.c  o  m*/
 * @param containers the list of {@link Container} returned by Yarn.
 */
@Override
public void onContainersAllocated(List<Container> containers) {
    List<SamzaResource> resources = new ArrayList<SamzaResource>();
    for (Container container : containers) {
        log.info("Got allocation notification for Container ID: {} on host: {}", container.getId(),
                container.getNodeId().getHost());
        String containerId = container.getId().toString();
        String host = container.getNodeId().getHost();
        int memory = container.getResource().getMemory();
        int numCores = container.getResource().getVirtualCores();

        SamzaResource resource = new SamzaResource(numCores, memory, host, containerId);
        allocatedResources.put(resource, container);
        resources.add(resource);
    }
    clusterManagerCallback.onResourcesAvailable(resources);
}

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

/**
 * Runs a process as specified by the command builder on the container.
 * @param processorId id of the samza processor to run (passed as a command line parameter to the process)
 * @param container the yarn container to run the processor on.
 * @param cmdBuilder the command builder that encapsulates the command, and the context
 * @throws IOException on IO exceptions running the container
 *//*from  ww w .ja  v a  2  s. c  o m*/
public void runProcessor(String processorId, Container container, CommandBuilder cmdBuilder)
        throws IOException {
    String containerIdStr = ConverterUtils.toString(container.getId());
    // check if we have framework path specified. If yes - use it, if not use default ./__package/
    String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries
    String cmdPath = "./__package/";

    String fwkPath = JobConfig.getFwkPath(this.config);
    if (fwkPath != null && (!fwkPath.isEmpty())) {
        cmdPath = fwkPath;
        jobLib = "export JOB_LIB_DIR=./__package/lib";
    }
    cmdBuilder.setCommandPath(cmdPath);
    String command = cmdBuilder.buildCommand();

    Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
    env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(),
            Util.envVarEscape(container.getId().toString()));

    Path packagePath = new Path(yarnConfig.getPackagePath());
    String formattedCommand = getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command,
            ApplicationConstants.STDOUT, ApplicationConstants.STDERR);

    log.info(
            "Running Processor ID: {} on Container ID: {} on host: {} using command: {} and env: {} and package path: {}",
            processorId, containerIdStr, container.getNodeHttpAddress(), formattedCommand, env, packagePath);
    state.pendingProcessors.put(processorId, new YarnContainer(container));

    startContainer(packagePath, container, env, formattedCommand);

    log.info(
            "Made start request for Processor ID: {} on Container ID: {} on host: {} (http://{}/node/containerlogs/{}).",
            processorId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(),
            containerIdStr);
}

From source file:org.apache.samza.job.yarn.YarnContainerRunner.java

License:Apache License

/**
 * Runs a process as specified by the command builder on the container.
 * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process)
 * @param container the samza container to run.
 * @param cmdBuilder the command builder that encapsulates the command, and the context
 *
 * @throws SamzaContainerLaunchException  when there's an exception in submitting the request to the RM.
 *
 *///  w  w w.ja  v a 2 s  .c o m
public void runContainer(String samzaContainerId, Container container, CommandBuilder cmdBuilder)
        throws SamzaContainerLaunchException {
    String containerIdStr = ConverterUtils.toString(container.getId());
    log.info("Got available container ID ({}) for container: {}", samzaContainerId, container);

    // check if we have framework path specified. If yes - use it, if not use default ./__package/
    String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries
    String cmdPath = "./__package/";

    String fwkPath = JobConfig.getFwkPath(config);
    if (fwkPath != null && (!fwkPath.isEmpty())) {
        cmdPath = fwkPath;
        jobLib = "export JOB_LIB_DIR=./__package/lib";
    }
    log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib);
    cmdBuilder.setCommandPath(cmdPath);

    String command = cmdBuilder.buildCommand();
    log.info("Container ID {} using command {}", samzaContainerId, command);

    Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder);
    env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(),
            Util.envVarEscape(container.getId().toString()));
    printContainerEnvironmentVariables(samzaContainerId, env);

    log.info("Samza FWK path: " + command + "; env=" + env);

    Path packagePath = new Path(yarnConfig.getPackagePath());
    log.info("Starting container ID {} using package path {}", samzaContainerId, packagePath);

    startContainer(packagePath, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
            jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR));

    log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).",
            new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(),
                    container.getNodeHttpAddress(), containerIdStr });

    log.info("Started container ID {}", samzaContainerId);
}