Example usage for org.apache.hadoop.yarn.api.records ContainerStatus getState

List of usage examples for org.apache.hadoop.yarn.api.records ContainerStatus getState

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ContainerStatus
  6. getState

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerStatus getState.

Prototype

@Public
@Stable
public abstract ContainerState getState();

Source Link

Document

Get the ContainerState of the container.

Usage

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

/**
 * Callback invoked from Yarn when containers complete. This translates the yarn callbacks into Samza specific
 * ones.//  ww w .j  a v  a  2 s.c  o  m
 *
 * @param statuses the YarnContainerStatus callbacks from Yarn.
 */
@Override
public void onContainersCompleted(List<ContainerStatus> statuses) {
    List<SamzaResourceStatus> samzaResourceStatuses = new ArrayList<>();

    for (ContainerStatus status : statuses) {
        log.info(
                "Got completion notification for Container ID: {} with status: {} and state: {}. Diagnostics information: {}.",
                status.getContainerId(), status.getExitStatus(), status.getState(), status.getDiagnostics());

        SamzaResourceStatus samzaResourceStatus = new SamzaResourceStatus(status.getContainerId().toString(),
                status.getDiagnostics(), status.getExitStatus());
        samzaResourceStatuses.add(samzaResourceStatus);

        String completedProcessorID = getRunningProcessorId(status.getContainerId().toString());
        log.info("Completed Container ID: {} had Processor ID: {}", status.getContainerId(),
                completedProcessorID);

        //remove the container from the list of running containers, if failed with a non-zero exit code, add it to the list of
        //failed containers.
        if (!completedProcessorID.equals(INVALID_PROCESSOR_ID)) {
            if (state.runningProcessors.containsKey(completedProcessorID)) {
                log.info("Removing Processor ID: {} from YarnClusterResourceManager running processors.",
                        completedProcessorID);
                state.runningProcessors.remove(completedProcessorID);

                if (status.getExitStatus() != ContainerExitStatus.SUCCESS)
                    state.failedContainersStatus.put(status.getContainerId().toString(), status);
            }
        }
    }
    clusterManagerCallback.onResourcesCompleted(samzaResourceStatuses);
}

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

@Override
public void onContainerStatusReceived(ContainerId containerId, ContainerStatus containerStatus) {
    log.info("Got status notification for Container ID: {} for Processor ID: {}. Status: {}", containerId,
            getRunningProcessorId(containerId.toString()), containerStatus.getState());
}

From source file:org.apache.slider.server.appmaster.SliderAppMaster.java

License:Apache License

@Override //AMRMClientAsync
public synchronized void onContainersCompleted(List<ContainerStatus> completedContainers) {
    LOG_YARN.info("onContainersCompleted([{}]", completedContainers.size());
    for (ContainerStatus status : completedContainers) {
        ContainerId containerId = status.getContainerId();
        LOG_YARN.info(//from w  w  w  .  j  a va2s .  co  m
                "Container Completion for" + " containerID={}," + " state={}," + " exitStatus={},"
                        + " diagnostics={}",
                containerId, status.getState(), status.getExitStatus(), status.getDiagnostics());

        // non complete containers should not be here
        assert (status.getState() == ContainerState.COMPLETE);
        AppState.NodeCompletionResult result = appState.onCompletedNode(status);
        if (result.containerFailed) {
            RoleInstance ri = result.roleInstance;
            log.error("Role instance {} failed ", ri);
        }

        //  known nodes trigger notifications
        if (!result.unknownNode) {
            getProviderService().notifyContainerCompleted(containerId);
            queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
        }
    }

    reviewRequestAndReleaseNodes("onContainersCompleted");
}

From source file:org.springframework.yarn.am.container.DefaultContainerLauncher.java

License:Apache License

/**
 * Contains the logic to do the actual polling.
 *
 * @return True if this poll operation did something, False otherwise
 *//* www  .  jav  a2 s  .c  om*/
private boolean doPoll() {
    boolean result = false;

    if (log.isDebugEnabled()) {
        log.debug("Checking status of containers previousely launched");
    }

    for (Iterator<Container> iterator = launched.iterator(); iterator.hasNext();) {
        Container container = iterator.next();
        ContainerStatus status = getCmTemplate(container).getContainerStatus();
        ContainerState state = status.getState();
        if (state.equals(ContainerState.COMPLETE)) {
            iterator.remove();
        } else if (state.equals(ContainerState.RUNNING)) {
            iterator.remove();
            if (getYarnEventPublisher() != null) {
                getYarnEventPublisher().publishContainerLaunched(this, container);
            }
        }
    }

    return result;
}

From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitor.java

License:Apache License

private void handleContainerStatus(List<ContainerStatus> containerStatuses, boolean notifyIntermediates) {
    for (ContainerStatus status : containerStatuses) {

        if (log.isDebugEnabled()) {
            log.debug("Reporting containerStatus=" + status);
        }/*  w w  w .jav a2 s .c  o m*/

        ContainerId containerId = status.getContainerId();
        int exitStatus = status.getExitStatus();
        ContainerState state = status.getState();
        String cid = ConverterUtils.toString(containerId);

        synchronized (lock) {
            if (state.equals(ContainerState.COMPLETE)) {
                if (exitStatus > 0) {
                    failed.add(cid);
                } else {
                    completed.add(cid);
                }
            }
            allocated.remove(cid);
            running.remove(cid);
        }

        if (notifyIntermediates) {
            dispatchCurrentContainerMonitorState();
        }
    }
    if (!notifyIntermediates) {
        dispatchCurrentContainerMonitorState();
    }
    if (log.isDebugEnabled()) {
        log.debug("State after handleContainerStatus: " + toDebugString());
    }
}

From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitorTests.java

License:Apache License

/**
 * Mock {@link ContainerStatus}// ww w . j a va2  s .  co m
 *
 * @param containerId the {@link ContainerId}
 * @param containerState the {@link ContainerState}
 * @param exitStatus the container exit status
 * @return mocked {@link ContainerStatus}
 */
public static ContainerStatus getMockContainerStatus(ContainerId containerId, ContainerState containerState,
        int exitStatus) {
    ContainerStatus status = mock(ContainerStatus.class);
    when(status.getContainerId()).thenReturn(containerId);
    when(status.getState()).thenReturn(containerState);
    when(status.getExitStatus()).thenReturn(exitStatus);
    return status;
}

From source file:org.springframework.yarn.boot.MockUtils.java

License:Apache License

public static ContainerStatus getMockContainerStatus(ContainerId containerId, ContainerState containerState,
        int exitStatus) {
    ContainerStatus status = mock(ContainerStatus.class);
    when(status.getContainerId()).thenReturn(containerId);
    when(status.getState()).thenReturn(containerState);
    when(status.getExitStatus()).thenReturn(exitStatus);
    return status;
}

From source file:runtime.starter.MPJAppMaster.java

License:Open Source License

public void run() throws Exception {
    try {/*  w w w . j a  va  2  s  . co  m*/
        appMasterSock = new Socket(serverName, ioServerPort);

        //redirecting stdout and stderr
        System.setOut(new PrintStream(appMasterSock.getOutputStream(), true));
        System.setErr(new PrintStream(appMasterSock.getOutputStream(), true));
    } catch (Exception exp) {
        exp.printStackTrace();
    }

    FileSystem fs = FileSystem.get(conf);
    Path wrapperDest = new Path(wrapperPath);
    FileStatus destStatus = fs.getFileStatus(wrapperDest);

    Path userFileDest = new Path(userJarPath);
    FileStatus destStatusClass = fs.getFileStatus(userFileDest);

    // Initialize AM <--> RM communication protocol
    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    // Initialize AM <--> NM communication protocol
    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    RegisterApplicationMasterResponse registerResponse = rmClient.registerApplicationMaster("", 0, "");
    // Priority for containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(mpjContainerPriority);

    maxMem = registerResponse.getMaximumResourceCapability().getMemory();

    if (debugYarn) {
        System.out.println("[MPJAppMaster]: Max memory capability resources " + "in cluster: " + maxMem);
    }

    if (containerMem > maxMem) {
        System.out.println("[MPJAppMaster]: container  memory specified above "
                + "threshold of cluster! Using maximum memory for " + "containers: " + containerMem);
        containerMem = maxMem;
    }

    maxCores = registerResponse.getMaximumResourceCapability().getVirtualCores();

    if (debugYarn) {
        System.out.println("[MPJAppMaster]: Max v-cores capability resources " + "in cluster: " + maxCores);
    }

    if (containerCores > maxCores) {
        System.out.println("[MPJAppMaster]: virtual cores specified above "
                + "threshold of cluster! Using maximum v-cores for " + "containers: " + containerCores);
        containerCores = maxCores;
    }

    // Resource requirements for containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMem);
    capability.setVirtualCores(containerCores);

    // Make container requests to ResourceManager
    for (int i = 0; i < np; ++i) {
        ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority);

        rmClient.addContainerRequest(containerReq);
    }

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    // Creating Local Resource for Wrapper   
    LocalResource wrapperJar = Records.newRecord(LocalResource.class);

    wrapperJar.setResource(ConverterUtils.getYarnUrlFromPath(wrapperDest));
    wrapperJar.setSize(destStatus.getLen());
    wrapperJar.setTimestamp(destStatus.getModificationTime());
    wrapperJar.setType(LocalResourceType.ARCHIVE);
    wrapperJar.setVisibility(LocalResourceVisibility.APPLICATION);

    // Creating Local Resource for UserClass
    LocalResource userClass = Records.newRecord(LocalResource.class);

    userClass.setResource(ConverterUtils.getYarnUrlFromPath(userFileDest));
    userClass.setSize(destStatusClass.getLen());
    userClass.setTimestamp(destStatusClass.getModificationTime());
    userClass.setType(LocalResourceType.ARCHIVE);
    userClass.setVisibility(LocalResourceVisibility.APPLICATION);

    localResources.put("mpj-yarn-wrapper.jar", wrapperJar);
    localResources.put("user-code.jar", userClass);

    while (allocatedContainers < np) {
        AllocateResponse response = rmClient.allocate(0);
        mpiContainers.addAll(response.getAllocatedContainers());
        allocatedContainers = mpiContainers.size();

        if (allocatedContainers != np) {
            Thread.sleep(100);
        }
    }

    if (debugYarn) {
        System.out.println("[MPJAppMaster]: launching " + allocatedContainers + " containers");
    }

    for (Container container : mpiContainers) {

        ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

        List<String> commands = new ArrayList<String>();

        commands.add(" $JAVA_HOME/bin/java");
        commands.add(" -Xmx" + containerMem + "m");
        commands.add(" runtime.starter.MPJYarnWrapper");
        commands.add("--serverName");
        commands.add(serverName); // server name
        commands.add("--ioServerPort");
        commands.add(Integer.toString(ioServerPort)); // IO server port
        commands.add("--deviceName");
        commands.add(deviceName); // device name
        commands.add("--className");
        commands.add(className); // class name
        commands.add("--psl");
        commands.add(psl); // protocol switch limit
        commands.add("--np");
        commands.add(Integer.toString(np)); // no. of containers
        commands.add("--rank");
        commands.add(" " + Integer.toString(rank++)); // rank

        //temp sock port to share rank and ports
        commands.add("--wireUpPort");
        commands.add(wireUpPort);

        if (appArgs != null) {
            commands.add("--appArgs");
            for (int i = 0; i < appArgs.length; i++) {
                commands.add(appArgs[i]);
            }
        }

        ctx.setCommands(commands);

        // Set local resource for containers
        ctx.setLocalResources(localResources);

        // Set environment for container
        Map<String, String> containerEnv = new HashMap<String, String>();
        setupEnv(containerEnv);
        ctx.setEnvironment(containerEnv);

        // Time to start the container
        nmClient.startContainer(container, ctx);

    }

    while (completedContainers < np) {
        // argument to allocate() is the progress indicator
        AllocateResponse response = rmClient.allocate(completedContainers / np);

        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            if (debugYarn) {
                System.out.println("\n[MPJAppMaster]: Container Id - " + status.getContainerId());
                System.out.println("[MPJAppMaster]: Container State - " + status.getState().toString());
                System.out.println("[MPJAppMaster]: Container Diagnostics - " + status.getDiagnostics());

            }
            ++completedContainers;
        }

        if (completedContainers != np) {
            Thread.sleep(100);
        }
        ;
    }
    // Un-register with ResourceManager 
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
    //shutDown AppMaster IO
    System.out.println("EXIT");
}