Example usage for org.apache.hadoop.yarn.api.records ContainerStatus getContainerId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ContainerStatus getContainerId.

Prototype

@Public
@Stable
public abstract ContainerId getContainerId();

Source Link

Document

Get the ContainerId of the container.

Usage

From source file:org.apache.tez.dag.app.rm.TestTaskSchedulerManager.java

License:Apache License

@Test(timeout = 5000)
public void testContainerDiskFailed() throws IOException {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);//from ww w.  j a  v  a  2 s .  co  m
    schedulerHandler.start();

    String diagnostics = "NM disk failed.";
    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
    ContainerStatus mockStatus = mock(ContainerStatus.class);
    ContainerId mockCId = mock(ContainerId.class);
    AMContainer mockAMContainer = mock(AMContainer.class);
    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
    when(mockStatus.getContainerId()).thenReturn(mockCId);
    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
    when(mockStatus.getExitStatus()).thenReturn(ContainerExitStatus.DISKS_FAILED);
    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
    assertEquals(1, mockEventHandler.events.size());
    Event event = mockEventHandler.events.get(0);
    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
    assertEquals(mockCId, completedEvent.getContainerId());
    assertEquals("Container disk failed. NM disk failed.", completedEvent.getDiagnostics());
    Assert.assertFalse(completedEvent.isPreempted());
    assertTrue(completedEvent.isDiskFailed());
    assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR, completedEvent.getTerminationCause());

    schedulerHandler.stop();
    schedulerHandler.close();
}

---

From source file:org.apache.tez.dag.app.rm.TestTaskSchedulerManager.java

License:Apache License

@Test(timeout = 5000)
public void testContainerExceededPMem() throws IOException {
    Configuration conf = new Configuration(false);
    schedulerHandler.init(conf);//from   www  . jav  a2 s . c o  m
    schedulerHandler.start();

    String diagnostics = "Exceeded Physical Memory";
    TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class);
    ContainerStatus mockStatus = mock(ContainerStatus.class);
    ContainerId mockCId = mock(ContainerId.class);
    AMContainer mockAMContainer = mock(AMContainer.class);
    when(mockAMContainerMap.get(mockCId)).thenReturn(mockAMContainer);
    when(mockAMContainer.getContainerId()).thenReturn(mockCId);
    when(mockStatus.getContainerId()).thenReturn(mockCId);
    when(mockStatus.getDiagnostics()).thenReturn(diagnostics);
    // use -104 rather than ContainerExitStatus.KILLED_EXCEEDED_PMEM because
    // ContainerExitStatus.KILLED_EXCEEDED_PMEM is only available after hadoop-2.5
    when(mockStatus.getExitStatus()).thenReturn(-104);
    schedulerHandler.containerCompleted(0, mockTask, mockStatus);
    assertEquals(1, mockEventHandler.events.size());
    Event event = mockEventHandler.events.get(0);
    assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
    AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
    assertEquals(mockCId, completedEvent.getContainerId());
    assertEquals("Container failed, exitCode=-104. Exceeded Physical Memory", completedEvent.getDiagnostics());
    Assert.assertFalse(completedEvent.isPreempted());
    Assert.assertFalse(completedEvent.isDiskFailed());
    assertEquals(TaskAttemptTerminationCause.CONTAINER_EXITED, completedEvent.getTerminationCause());

    schedulerHandler.stop();
    schedulerHandler.close();
}

---

From source file:org.apache.tez.dag.app.rm.YarnTaskSchedulerService.java

License:Apache License

@Override
public void onContainersCompleted(List<ContainerStatus> statuses) {
    if (isStopped.get()) {
        return;/*www  .  j a v a 2s .co m*/
    }
    Map<Object, ContainerStatus> appContainerStatus = new HashMap<Object, ContainerStatus>(statuses.size());
    synchronized (this) {
        for (ContainerStatus containerStatus : statuses) {
            ContainerId completedId = containerStatus.getContainerId();
            HeldContainer delayedContainer = heldContainers.get(completedId);

            Object task = releasedContainers.remove(completedId);
            if (task != null) {
                if (delayedContainer != null) {
                    LOG.warn("Held container should be null since releasedContainer is not");
                }
                // TODO later we may want to check if exit code matched expectation
                // e.g. successful container should not come back fail exit code after
                // being released
                // completion of a container we had released earlier
                // an allocated container completed. notify app
                LOG.info("Released container completed:" + completedId + " last allocated to task: " + task);
                appContainerStatus.put(task, containerStatus);
                continue;
            }

            // not found in released containers. check currently allocated containers
            // no need to release this container as the RM has already completed it
            task = unAssignContainer(completedId, false);
            if (delayedContainer != null) {
                heldContainers.remove(completedId);
                Resources.subtract(allocatedResources, delayedContainer.getContainer().getResource());
            } else {
                LOG.warn("Held container expected to be not null for a non-AM-released container");
            }
            if (task != null) {
                // completion of a container we have allocated currently
                // an allocated container completed. notify app
                LOG.info("Allocated container completed:" + completedId + " last allocated to task: " + task);
                appContainerStatus.put(task, containerStatus);
                continue;
            }

            // container neither allocated nor released
            LOG.info("Ignoring unknown container: " + containerStatus.getContainerId());
        }
    }

    // upcall to app must be outside locks
    for (Entry<Object, ContainerStatus> entry : appContainerStatus.entrySet()) {
        appClientDelegate.containerCompleted(entry.getKey(), entry.getValue());
    }
}

---

From source file:org.chenchun.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws IOException, YarnException, InterruptedException {
    final String params = args[0];
    final int containerNum = Integer.valueOf(args[1]);

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<AMRMClient.ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);/*from w w  w  .  ja  va2s .  c  o m*/
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    System.out.println("registerApplicationMaster 0");
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("registerApplicationMaster 1");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i = 0; i < containerNum; ++i) {
        AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null,
                priority);
        System.out.println("Making res-req " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    while (completedContainers < containerNum) {
        AllocateResponse response = rmClient.allocate(responseId++);
        System.out.println("Allocate response " + response.getAMCommand() + " " + "allocate "
                + response.getAllocatedContainers().size() + "contains");
        for (Container container : response.getAllocatedContainers()) {
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(params + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                            + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));
            System.out.println("Launching container " + container.getId() + " on " + container.getNodeId());
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            System.out.println("Completed container " + status.getContainerId());
        }
        Thread.sleep(1000);
    }
    System.out.println("Unregister ApplicationMaster");

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

---

From source file:org.elasticsearch.hadoop.yarn.am.EsCluster.java

License:Apache License

public void start() {
    running = true;//from   w  w  w.j av a  2 s .c om
    nmRpc.start();

    UserGroupInformation.setConfiguration(cfg);

    log.info(String.format("Allocating Elasticsearch cluster with %d nodes", appConfig.containersToAllocate()));

    // register requests
    Resource capability = YarnCompat.resource(cfg, appConfig.containerMem(), appConfig.containerVCores());
    Priority prio = Priority.newInstance(appConfig.amPriority());

    for (int i = 0; i < appConfig.containersToAllocate(); i++) {
        // TODO: Add allocation (host/rack rules) - and disable location constraints
        ContainerRequest req = new ContainerRequest(capability, null, null, prio);
        amRpc.addContainerRequest(req);
    }

    // update status every 5 sec
    final long heartBeatRate = TimeUnit.SECONDS.toMillis(5);

    // start the allocation loop
    // when a new container is allocated, launch it right away

    int responseId = 0;

    try {
        do {
            AllocateResponse alloc = amRpc.allocate(responseId++);
            List<Container> currentlyAllocated = alloc.getAllocatedContainers();
            for (Container container : currentlyAllocated) {
                launchContainer(container);
                allocatedContainers.add(container.getId());
            }

            if (currentlyAllocated.size() > 0) {
                int needed = appConfig.containersToAllocate() - allocatedContainers.size();
                if (needed > 0) {
                    log.info(String.format("%s containers allocated, %s remaining", allocatedContainers.size(),
                            needed));
                } else {
                    log.info(String.format("Fully allocated %s containers", allocatedContainers.size()));
                }
            }

            List<ContainerStatus> completed = alloc.getCompletedContainersStatuses();
            for (ContainerStatus status : completed) {
                if (!completedContainers.contains(status.getContainerId())) {
                    ContainerId containerId = status.getContainerId();
                    completedContainers.add(containerId);

                    boolean containerSuccesful = false;

                    switch (status.getExitStatus()) {
                    case ContainerExitStatus.SUCCESS:
                        log.info(String.format("Container %s finished succesfully...", containerId));
                        containerSuccesful = true;
                        break;
                    case ContainerExitStatus.ABORTED:
                        log.warn(String.format("Container %s aborted...", containerId));
                        break;
                    case ContainerExitStatus.DISKS_FAILED:
                        log.warn(String.format("Container %s ran out of disk...", containerId));
                        break;
                    case ContainerExitStatus.PREEMPTED:
                        log.warn(String.format("Container %s preempted...", containerId));
                        break;
                    default:
                        log.warn(String.format("Container %s exited with an invalid/unknown exit code...",
                                containerId));
                    }

                    if (!containerSuccesful) {
                        log.warn("Cluster has not completed succesfully...");
                        clusterHasFailed = true;
                        running = false;
                    }
                }
            }

            if (completedContainers.size() == appConfig.containersToAllocate()) {
                running = false;
            }

            if (running) {
                try {
                    Thread.sleep(heartBeatRate);
                } catch (Exception ex) {
                    throw new EsYarnNmException("Cluster interrupted");
                }
            }
        } while (running);
    } finally {
        log.info("Cluster has completed running...");
        try {
            Thread.sleep(TimeUnit.SECONDS.toMillis(15));
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }
        close();
    }
}

---

From source file:org.springframework.cloud.deployer.spi.yarn.appdeployer.StreamAppmaster.java

License:Apache License

@Override
protected void onContainerCompleted(ContainerStatus status) {
    super.onContainerCompleted(status);
    String containerClusterId = containerIdMap.get(status.getContainerId());
    if (containerClusterId != null) {
        synchronized (indexTracker) {
            indexTracker.freeIndex(status.getContainerId(), containerClusterId);
        }/* w ww.  j  a va  2 s.  c  o  m*/
    }
}

---

From source file:org.springframework.yarn.am.allocate.AbstractPollingAllocator.java

License:Apache License

/**
 * Contains the logic to do the actual polling.
 *
 * @return True if this poll operation did something, False otherwise
 */// w  w w .ja  v  a  2s  . co m
private boolean doPoll() {
    boolean result = false;

    if (log.isDebugEnabled()) {
        log.debug("Checking if we can poll new and completed containers.");
    }

    // we use application attempt id as a flag
    // to know when appmaster has done registration
    if (getApplicationAttemptId() == null) {
        if (log.isDebugEnabled()) {
            log.debug("ApplicationAttemptId not set, delaying poll requests.");
        }
        return result;
    }

    AllocateResponse response = doContainerRequest();

    // for now just stash tokens into hadoops NMTokenCache
    if (!response.getNMTokens().isEmpty()) {
        populateNmTokenCache(response);
    }

    List<Container> allocatedContainers = preProcessAllocatedContainers(response.getAllocatedContainers());
    if (allocatedContainers != null && allocatedContainers.size() > 0) {
        if (log.isDebugEnabled()) {
            log.debug("response has " + allocatedContainers.size() + " new containers");
            for (Container c : allocatedContainers) {
                log.debug("new container: " + c.getId());
            }
        }
        handleAllocatedContainers(allocatedContainers);
        if (getYarnEventPublisher() != null) {
            for (Container container : allocatedContainers) {
                getYarnEventPublisher().publishContainerAllocated(this, container);
            }
        }
        result = true;
    }

    List<ContainerStatus> containerStatuses = response.getCompletedContainersStatuses();
    if (containerStatuses != null && containerStatuses.size() > 0) {
        if (log.isDebugEnabled()) {
            log.debug("response has " + containerStatuses.size() + " completed containers");
            for (ContainerStatus cs : containerStatuses) {
                log.debug("completed container: " + cs.getContainerId() + " with status=" + cs);
            }
        }
        handleCompletedContainers(containerStatuses);
        if (getYarnEventPublisher() != null) {
            for (ContainerStatus containerStatus : containerStatuses) {
                getYarnEventPublisher().publishContainerCompleted(this, containerStatus);
            }
        }
        result = true;
    }

    return result;
}

---

From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocator.java

License:Apache License

@Override
protected void handleCompletedContainers(List<ContainerStatus> containerStatuses) {
    // strip away containers which were already marked
    // garbage by allocate tracker. system
    // never knew those even exist and might create mess
    // with monitor component. monitor only sees
    // complete status which is also the case for garbage
    // when it's released.
    List<ContainerStatus> garbageFree = new ArrayList<ContainerStatus>();
    for (ContainerStatus status : containerStatuses) {
        if (!garbageContainers.contains(status.getContainerId())) {
            garbageFree.add(status);//from  w w  w. j a va2s .  c  o m
        }
    }
    allocatorListener.completed(garbageFree);
}

---

From source file:org.springframework.yarn.am.cluster.AbstractContainerClusterAppmaster.java

License:Apache License

@Override
protected void onContainerCompleted(ContainerStatus status) {
    super.onContainerCompleted(status);

    boolean removed = grid.removeMember(status.getContainerId());
    if (!removed) {
        // force allocation for all clusters if we got completed
        // container unknown to a grid. might be an indication
        // that satisfy states are not met.
        requestAllocationForAll();//from  w  w  w  . j  a  v  a  2  s.co  m
    }

    if (getMonitor() instanceof ContainerAware) {
        ((ContainerAware) getMonitor()).onContainerStatus(Arrays.asList(status));
    }
}

---

From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitor.java

License:Apache License

private void handleContainerStatus(List<ContainerStatus> containerStatuses, boolean notifyIntermediates) {
    for (ContainerStatus status : containerStatuses) {

        if (log.isDebugEnabled()) {
            log.debug("Reporting containerStatus=" + status);
        }/*from www.  ja v a2s  .co m*/

        ContainerId containerId = status.getContainerId();
        int exitStatus = status.getExitStatus();
        ContainerState state = status.getState();
        String cid = ConverterUtils.toString(containerId);

        synchronized (lock) {
            if (state.equals(ContainerState.COMPLETE)) {
                if (exitStatus > 0) {
                    failed.add(cid);
                } else {
                    completed.add(cid);
                }
            }
            allocated.remove(cid);
            running.remove(cid);
        }

        if (notifyIntermediates) {
            dispatchCurrentContainerMonitorState();
        }
    }
    if (!notifyIntermediates) {
        dispatchCurrentContainerMonitorState();
    }
    if (log.isDebugEnabled()) {
        log.debug("State after handleContainerStatus: " + toDebugString());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »