Example usage for org.apache.hadoop.yarn.api.records Container getNodeId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records Container getNodeId.

Prototype

@Public
@Stable
public abstract NodeId getNodeId();

Source Link

Document

Get the identifier of the node on which the container is allocated.

Usage

From source file:org.apache.slider.providers.agent.AgentProviderService.java

License:Apache License

private void localizeContainerSSLResources(ContainerLauncher launcher, Container container,
        SliderFileSystem fileSystem) throws SliderException {
    try {//w  w  w. j a  v a 2 s .  c om
        // localize server cert
        Path certsDir = new Path(fileSystem.buildClusterDirPath(getClusterName()), "certs");
        LocalResource certResource = fileSystem.createAmResource(new Path(certsDir, SliderKeys.CRT_FILE_NAME),
                LocalResourceType.FILE);
        launcher.addLocalResource(AgentKeys.CERT_FILE_LOCALIZATION_PATH, certResource);

        // generate and localize agent cert
        CertificateManager certMgr = new CertificateManager();
        String hostname = container.getNodeId().getHost();
        String containerId = container.getId().toString();
        certMgr.generateAgentCertificate(hostname, containerId);
        LocalResource agentCertResource = fileSystem.createAmResource(
                uploadSecurityResource(CertificateManager.getAgentCertficateFilePath(containerId), fileSystem),
                LocalResourceType.FILE);
        // still using hostname as file name on the agent side, but the files
        // do end up under the specific container's file space
        launcher.addLocalResource("certs/" + hostname + ".crt", agentCertResource);
        LocalResource agentKeyResource = fileSystem.createAmResource(
                uploadSecurityResource(CertificateManager.getAgentKeyFilePath(containerId), fileSystem),
                LocalResourceType.FILE);
        launcher.addLocalResource("certs/" + hostname + ".key", agentKeyResource);

    } catch (Exception e) {
        throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
                "Unable to localize certificates.  Two-way SSL cannot be enabled");
    }
}

---

From source file:org.apache.slider.providers.agent.TestAgentProviderService.java

License:Apache License

@Test
public void testRegistration() throws IOException {

    ConfTree tree = new ConfTree();
    tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");

    AgentProviderService aps = createAgentProviderService(new Configuration());
    ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
    AggregateConf instanceDefinition = new AggregateConf();

    instanceDefinition.setInternal(tree);
    instanceDefinition.setAppConf(tree);
    instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
    instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
    instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");

    Container container = createNiceMock(Container.class);
    String role = "HBASE_MASTER";
    SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
    ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
    Path generatedConfPath = new Path(".", "test");
    MapOperations resourceComponent = new MapOperations();
    MapOperations appComponent = new MapOperations();
    Path containerTmpDirPath = new Path(".", "test");
    FileSystem mockFs = createNiceMock(FileSystem.class);
    expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
    expect(sliderFileSystem.getFileSystem()).andReturn(mockFs).anyTimes();
    expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class)))
            .andReturn(createNiceMock(LocalResource.class)).anyTimes();
    expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
    expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes();
    StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);

    AgentProviderService mockAps = Mockito.spy(aps);
    doReturn(access).when(mockAps).getAmState();
    CommandScript cs = new CommandScript();
    cs.setScript("scripts/hbase_master.py");
    doReturn(cs).when(mockAps).getScriptPathFromMetainfo(anyString());
    Metainfo metainfo = new Metainfo();
    metainfo.setApplication(new Application());
    doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());

    Configuration conf = new Configuration();
    conf.set(RegistryConstants.KEY_REGISTRY_ZK_ROOT, RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);

    try {/*from  ww w  .  j  a v  a  2  s .c  o  m*/
        doReturn(true).when(mockAps).isMaster(anyString());
        doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"), eq("mockcontainer_1"),
                any(HeartBeatResponse.class), eq("scripts/hbase_master.py"), eq(600L));
        doReturn(conf).when(mockAps).getConfig();
    } catch (SliderException e) {
    }

    doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(), anyString(), anyMap());

    doNothing().when(mockAps).publishFolderPaths(anyMap(), anyString(), anyString(), anyString());
    expect(access.isApplicationLive()).andReturn(true).anyTimes();
    ClusterDescription desc = new ClusterDescription();
    desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
    desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
    expect(access.getClusterStatus()).andReturn(desc).anyTimes();

    AggregateConf aggConf = new AggregateConf();
    ConfTreeOperations treeOps = aggConf.getAppConfOperations();
    treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
    treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
    expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
    expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
    replay(access, ctx, container, sliderFileSystem, mockFs);

    try {
        mockAps.buildContainerLaunchContext(launcher, instanceDefinition, container, role, sliderFileSystem,
                generatedConfPath, resourceComponent, appComponent, containerTmpDirPath);
        // JDK7
    } catch (IOException he) {
        log.warn("{}", he, he);
    } catch (SliderException he) {
        log.warn("{}", he, he);
    }

    Register reg = new Register();
    reg.setResponseId(0);
    reg.setLabel("mockcontainer_1___HBASE_MASTER");
    Map<String, String> ports = new HashMap<String, String>();
    ports.put("a", "100");
    reg.setAllocatedPorts(ports);
    Map<String, String> folders = new HashMap<String, String>();
    folders.put("F1", "F2");
    reg.setLogFolders(folders);
    RegistrationResponse resp = mockAps.handleRegistration(reg);
    Assert.assertEquals(0, resp.getResponseId());
    Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());

    Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(anyString(), anyString(), anyString(),
            anyMap());

    Mockito.verify(mockAps, Mockito.times(1)).publishFolderPaths(anyMap(), anyString(), anyString(),
            anyString());

    HeartBeat hb = new HeartBeat();
    hb.setResponseId(1);
    hb.setHostname("mockcontainer_1___HBASE_MASTER");
    HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
    Assert.assertEquals(2, hbr.getResponseId());
}

---

From source file:org.apache.slider.providers.agent.TestAgentProviderService.java

License:Apache License

private AgentProviderService prepareProviderServiceForAgentStateTests() throws IOException {
    ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
    Container container = createNiceMock(Container.class);
    String role = "HBASE_MASTER";
    SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
    FileSystem mockFs = new MockFileSystem();
    expect(sliderFileSystem.getFileSystem()).andReturn(new FilterFileSystem(mockFs)).anyTimes();
    expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class)))
            .andReturn(createNiceMock(LocalResource.class)).anyTimes();
    expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
    expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes();
    expect(container.getPriority()).andReturn(Priority.newInstance(1));

    StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
    Configuration conf = new Configuration();

    AgentProviderService aps = createAgentProviderService(conf);
    AgentProviderService mockAps = Mockito.spy(aps);

    doReturn(access).when(mockAps).getAmState();
    CommandScript cs = new CommandScript();
    cs.setScript("scripts/hbase_master.py");
    doReturn(cs).when(mockAps).getScriptPathFromMetainfo(anyString());
    Metainfo metainfo = new Metainfo();
    Application application = new Application();
    metainfo.setApplication(application);
    doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
    doReturn(metainfo).when(mockAps).getMetainfo();

    try {/*from   w  w  w. j a va2s. co m*/
        doReturn(true).when(mockAps).isMaster(anyString());
        doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"), eq("mockcontainer_1"),
                any(HeartBeatResponse.class), eq("scripts/hbase_master.py"), eq(600L));
        doReturn(conf).when(mockAps).getConfig();
    } catch (SliderException e) {
    }

    doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(), anyString(), anyMap());
    expect(access.isApplicationLive()).andReturn(true).anyTimes();
    ClusterDescription desc = new ClusterDescription();
    desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
    desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
    expect(access.getClusterStatus()).andReturn(desc).anyTimes();

    AggregateConf aggConf = new AggregateConf();
    ConfTreeOperations treeOps = aggConf.getAppConfOperations();
    treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
    treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
    treeOps.set("java_home", "/usr/jdk7/");
    treeOps.set("site.fs.defaultFS", "hdfs://c6409.ambari.apache.org:8020");
    treeOps.set(InternalKeys.INTERNAL_DATA_DIR_PATH,
            "hdfs://c6409.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/data");
    expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
    expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
    expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
    replay(access, ctx, container, sliderFileSystem);

    List<Container> containers = new ArrayList<Container>();
    containers.add(container);
    Map<Integer, ProviderRole> providerRoleMap = new HashMap<Integer, ProviderRole>();
    ProviderRole providerRole = new ProviderRole(role, 1);
    providerRoleMap.put(1, providerRole);
    mockAps.rebuildContainerDetails(containers, "mockcontainer_1", providerRoleMap);
    return mockAps;
}

---

From source file:org.apache.slider.providers.agent.TestAgentProviderService.java

License:Apache License

@Test
public void testOrchestratedAppStart() throws IOException {
    // App has two components HBASE_MASTER and HBASE_REGIONSERVER
    // Start of HBASE_RS depends on the start of HBASE_MASTER
    InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
    Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
    ConfTree tree = new ConfTree();
    tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");

    Configuration conf = new Configuration();
    AgentProviderService aps = createAgentProviderService(conf);
    YarnRegistryViewForProviders registryViewForProviders = aps.getYarnRegistry();
    assertNotNull(registryViewForProviders);

    ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
    AggregateConf instanceDefinition = new AggregateConf();

    instanceDefinition.setInternal(tree);
    instanceDefinition.setAppConf(tree);
    instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
    instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
    instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");

    Container container = createNiceMock(Container.class);
    String role_hm = "HBASE_MASTER";
    String role_hrs = "HBASE_REGIONSERVER";
    SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
    ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
    ContainerLauncher launcher2 = createNiceMock(ContainerLauncher.class);
    Path generatedConfPath = new Path(".", "test");
    MapOperations resourceComponent = new MapOperations();
    MapOperations appComponent = new MapOperations();
    Path containerTmpDirPath = new Path(".", "test");
    FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
    expect(sliderFileSystem.getFileSystem()).andReturn(mockFs).anyTimes();
    expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
    expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class)))
            .andReturn(createNiceMock(LocalResource.class)).anyTimes();
    expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
    expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes();
    StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);

    AgentProviderService mockAps = Mockito.spy(aps);
    doReturn(access).when(mockAps).getAmState();
    doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
    doReturn(new HashMap<String, DefaultConfig>()).when(mockAps)
            .initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class));

    try {/*from w  w w.jav a  2s.  co m*/
        doReturn(true).when(mockAps).isMaster(anyString());
        doNothing().when(mockAps).addInstallCommand(anyString(), anyString(), any(HeartBeatResponse.class),
                anyString(), Mockito.anyLong());
        doNothing().when(mockAps).addStartCommand(anyString(), anyString(), any(HeartBeatResponse.class),
                anyString(), Mockito.anyLong(), Matchers.anyBoolean());
        doNothing().when(mockAps).addGetConfigCommand(anyString(), anyString(), any(HeartBeatResponse.class));
        doNothing().when(mockAps).publishFolderPaths(anyMap(), anyString(), anyString(), anyString());
        doReturn(conf).when(mockAps).getConfig();
    } catch (SliderException e) {
    }

    PublishedExportsSet pubExpSet = new PublishedExportsSet();
    expect(access.isApplicationLive()).andReturn(true).anyTimes();
    expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
    ClusterDescription desc = new ClusterDescription();
    desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
    desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
    expect(access.getClusterStatus()).andReturn(desc).anyTimes();

    AggregateConf aggConf = new AggregateConf();
    ConfTreeOperations treeOps = aggConf.getAppConfOperations();
    treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
    treeOps.getOrAddComponent("HBASE_REGIONSERVER").put(AgentKeys.WAIT_HEARTBEAT, "0");
    treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
    expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes();
    expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
    doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
    replay(access, ctx, container, sliderFileSystem, mockFs);

    // build two containers
    try {
        mockAps.buildContainerLaunchContext(launcher, instanceDefinition, container, role_hm, sliderFileSystem,
                generatedConfPath, resourceComponent, appComponent, containerTmpDirPath);

        mockAps.buildContainerLaunchContext(launcher2, instanceDefinition, container, role_hrs,
                sliderFileSystem, generatedConfPath, resourceComponent, appComponent, containerTmpDirPath);

        // Both containers register
        Register reg = new Register();
        reg.setResponseId(0);
        reg.setLabel("mockcontainer_1___HBASE_MASTER");
        RegistrationResponse resp = mockAps.handleRegistration(reg);
        Assert.assertEquals(0, resp.getResponseId());
        Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());

        reg = new Register();
        reg.setResponseId(0);
        reg.setLabel("mockcontainer_1___HBASE_REGIONSERVER");
        resp = mockAps.handleRegistration(reg);
        Assert.assertEquals(0, resp.getResponseId());
        Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());

        // Both issue install command
        HeartBeat hb = new HeartBeat();
        hb.setResponseId(1);
        hb.setHostname("mockcontainer_1___HBASE_MASTER");
        HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(2, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(1)).addInstallCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong());

        hb = new HeartBeat();
        hb.setResponseId(1);
        hb.setHostname("mockcontainer_1___HBASE_REGIONSERVER");
        hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(2, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(2)).addInstallCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong());
        // RS succeeds install but does not start
        hb = new HeartBeat();
        hb.setResponseId(2);
        hb.setHostname("mockcontainer_1___HBASE_REGIONSERVER");
        CommandReport cr = new CommandReport();
        cr.setRole("HBASE_REGIONSERVER");
        cr.setRoleCommand("INSTALL");
        cr.setStatus("COMPLETED");
        cr.setFolders(new HashMap<String, String>() {
            {
                put("a", "b");
            }
        });
        hb.setReports(Arrays.asList(cr));
        hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(3, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong(), Matchers.anyBoolean());
        // RS still does not start
        hb = new HeartBeat();
        hb.setResponseId(3);
        hb.setHostname("mockcontainer_1___HBASE_REGIONSERVER");
        hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(4, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong(), Matchers.anyBoolean());

        // MASTER succeeds install and issues start
        hb = new HeartBeat();
        hb.setResponseId(2);
        hb.setHostname("mockcontainer_1___HBASE_MASTER");
        hb.setFqdn("host1");
        cr = new CommandReport();
        cr.setRole("HBASE_MASTER");
        cr.setRoleCommand("INSTALL");
        cr.setStatus("COMPLETED");
        Map<String, String> ap = new HashMap<String, String>();
        ap.put("a.port", "10233");
        cr.setAllocatedPorts(ap);
        hb.setReports(Arrays.asList(cr));
        hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(3, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong(), Matchers.anyBoolean());
        Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
        Assert.assertTrue(allocatedPorts != null);
        Assert.assertTrue(allocatedPorts.size() == 1);
        Assert.assertTrue(allocatedPorts.containsKey("a.port"));

        // RS still does not start
        hb = new HeartBeat();
        hb.setResponseId(4);
        hb.setHostname("mockcontainer_1___HBASE_REGIONSERVER");
        hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(5, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong(), Matchers.anyBoolean());
        // MASTER succeeds start
        hb = new HeartBeat();
        hb.setResponseId(3);
        hb.setHostname("mockcontainer_1___HBASE_MASTER");
        cr = new CommandReport();
        cr.setRole("HBASE_MASTER");
        cr.setRoleCommand("START");
        cr.setStatus("COMPLETED");
        hb.setReports(Arrays.asList(cr));
        mockAps.handleHeartBeat(hb);
        Mockito.verify(mockAps, Mockito.times(1)).addGetConfigCommand(anyString(), anyString(),
                any(HeartBeatResponse.class));

        // RS starts now
        hb = new HeartBeat();
        hb.setResponseId(5);
        hb.setHostname("mockcontainer_1___HBASE_REGIONSERVER");
        hbr = mockAps.handleHeartBeat(hb);
        Assert.assertEquals(6, hbr.getResponseId());
        Mockito.verify(mockAps, Mockito.times(2)).addStartCommand(anyString(), anyString(),
                any(HeartBeatResponse.class), anyString(), Mockito.anyLong(), Matchers.anyBoolean());
        // JDK7 
    } catch (SliderException he) {
        log.warn(he.getMessage());
    } catch (IOException he) {
        log.warn(he.getMessage());
    }

    Mockito.verify(mockAps, Mockito.times(1)).publishFolderPaths(anyMap(), anyString(), anyString(),
            anyString());
}

---

From source file:org.apache.slider.server.appmaster.state.AppState.java

License:Apache License

/**
 * handle completed node in the CD -move something from the live
 * server list to the completed server list
 * @param status the node that has just completed
 * @return NodeCompletionResult/*from w  w  w. j a  v  a 2s  .c  o m*/
 */
public synchronized NodeCompletionResult onCompletedNode(ContainerStatus status) {
    ContainerId containerId = status.getContainerId();
    NodeCompletionResult result = new NodeCompletionResult();
    RoleInstance roleInstance;

    if (containersBeingReleased.containsKey(containerId)) {
        log.info("Container was queued for release : {}", containerId);
        Container container = containersBeingReleased.remove(containerId);
        RoleStatus roleStatus = lookupRoleStatus(container);
        int releasing = roleStatus.decReleasing();
        int actual = roleStatus.decActual();
        int completedCount = roleStatus.incCompleted();
        log.info("decrementing role count for role {} to {}; releasing={}, completed={}", roleStatus.getName(),
                actual, releasing, completedCount);
        roleHistory.onReleaseCompleted(container, true);

    } else if (surplusNodes.remove(containerId)) {
        //its a surplus one being purged
        result.surplusNode = true;
    } else {
        //a container has failed 
        result.containerFailed = true;
        roleInstance = removeOwnedContainer(containerId);
        if (roleInstance != null) {
            //it was active, move it to failed 
            incFailedCountainerCount();
            failedNodes.put(containerId, roleInstance);
        } else {
            // the container may have been noted as failed already, so look
            // it up
            roleInstance = failedNodes.get(containerId);
        }
        if (roleInstance != null) {
            int roleId = roleInstance.roleId;
            String rolename = roleInstance.role;
            log.info("Failed container in role[{}] : {}", roleId, rolename);
            try {
                RoleStatus roleStatus = lookupRoleStatus(roleId);
                roleStatus.decActual();
                boolean shortLived = isShortLived(roleInstance);
                String message;
                Container failedContainer = roleInstance.container;

                //build the failure message
                if (failedContainer != null) {
                    String completedLogsUrl = getLogsURLForContainer(failedContainer);
                    message = String.format("Failure %s on host %s: %s",
                            roleInstance.getContainerId().toString(), failedContainer.getNodeId().getHost(),
                            completedLogsUrl);
                } else {
                    message = String.format("Failure %s", containerId);
                }
                int failed = roleStatus.noteFailed(shortLived, message);
                log.info("Current count of failed role[{}] {} =  {}", roleId, rolename, failed);
                if (failedContainer != null) {
                    roleHistory.onFailedContainer(failedContainer, shortLived);
                }

            } catch (YarnRuntimeException e1) {
                log.error("Failed container of unknown role {}", roleId);
            }
        } else {
            //this isn't a known container.

            log.error("Notified of completed container {} that is not in the list"
                    + " of active or failed containers", containerId);
            completionOfUnknownContainerEvent.incrementAndGet();
            result.unknownNode = true;
        }
    }

    if (result.surplusNode) {
        //a surplus node
        return result;
    }

    //record the complete node's details; this pulls it from the livenode set 
    //remove the node
    ContainerId id = status.getContainerId();
    log.info("Removing node ID {}", id);
    RoleInstance node = getLiveNodes().remove(id);
    if (node != null) {
        node.state = ClusterDescription.STATE_DESTROYED;
        node.exitCode = status.getExitStatus();
        node.diagnostics = status.getDiagnostics();
        getCompletedNodes().put(id, node);
        result.roleInstance = node;
    } else {
        // not in the list
        log.warn("Received notification of completion of unknown node {}", id);
        completionOfNodeNotInLiveListEvent.incrementAndGet();

    }

    // and the active node list if present
    removeOwnedContainer(containerId);

    // finally, verify the node doesn't exist any more
    assert !containersBeingReleased.containsKey(containerId) : "container still in release queue";
    assert !getLiveNodes().containsKey(containerId) : " container still in live nodes";
    assert getOwnedContainer(containerId) == null : "Container still in active container list";

    return result;
}

---

From source file:org.apache.slider.server.appmaster.state.AppState.java

License:Apache License

/**
 * Get the URL log for a container//from  w w w .  j a v  a  2s.  com
 * @param c container
 * @return the URL or "" if it cannot be determined
 */
protected String getLogsURLForContainer(Container c) {
    if (c == null) {
        return null;
    }
    String user = null;
    try {
        user = SliderUtils.getCurrentUser().getShortUserName();
    } catch (IOException ignored) {
    }
    String completedLogsUrl = "";
    String url = logServerURL;
    if (user != null && SliderUtils.isSet(url)) {
        completedLogsUrl = url + "/" + c.getNodeId() + "/" + c.getId() + "/ctx/" + user;
    }
    return completedLogsUrl;
}

---

From source file:org.apache.slider.server.appmaster.state.AppState.java

License:Apache License

/**
 * Add a restarted container by walking it through the create/submit/start
 * lifecycle, so building up the internal structures
 * @param container container that was running before the AM restarted
 * @throws RuntimeException on problems// w  w  w .ja v a 2 s  . c  om
 */
private void addRestartedContainer(Container container) throws BadClusterStateException {
    String containerHostInfo = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
    // get the container ID
    ContainerId cid = container.getId();

    // get the role
    int roleId = ContainerPriority.extractRole(container);
    RoleStatus role = lookupRoleStatus(roleId);
    // increment its count
    role.incActual();
    String roleName = role.getName();

    log.info("Rebuilding container {} in role {} on {},", cid, roleName, containerHostInfo);

    //update app state internal structures and maps

    RoleInstance instance = new RoleInstance(container);
    instance.command = roleName;
    instance.role = roleName;
    instance.roleId = roleId;
    instance.environment = new String[0];
    instance.container = container;
    instance.createTime = now();
    instance.state = ClusterDescription.STATE_LIVE;
    putOwnedContainer(cid, instance);
    //role history gets told
    roleHistory.onContainerAssigned(container);
    // pretend the container has just had its start actions submitted
    containerStartSubmitted(container, instance);
    // now pretend it has just started
    innerOnNodeManagerContainerStarted(cid);
}

---

From source file:org.apache.slider.server.appmaster.state.RoleHistoryUtils.java

License:Apache License

public static String hostnameOf(Container container) {
    NodeId nodeId = container.getNodeId();
    if (nodeId == null) {
        throw new RuntimeException("Container has no node ID: %s" + SliderUtils.containerToString(container));
    }//  www .jav a2s.  co  m
    return nodeId.getHost();
}

---

From source file:org.apache.slider.server.appmaster.state.RoleInstance.java

License:Apache License

public RoleInstance(Container container) {
    Preconditions.checkNotNull(container, "Null container");
    Preconditions.checkState(container.getId() != null, "Null container ID");

    this.container = container;
    id = container.getId().toString();/*from   w w  w .  j a v a  2  s.  co  m*/
    if (container.getNodeId() != null) {
        host = container.getNodeId().getHost();
    }
    if (container.getNodeHttpAddress() != null) {
        hostURL = "http://" + container.getNodeHttpAddress();
    }
}

---

From source file:org.apache.tajo.master.rm.RMContainerAllocator.java

License:Apache License

public void heartbeat() throws Exception {
    AllocateResponse allocateResponse = allocate(context.getProgress());
    AMResponse response = allocateResponse.getAMResponse();
    List<Container> allocatedContainers = response.getAllocatedContainers();

    LOG.info("Available Cluster Nodes: " + allocateResponse.getNumClusterNodes());
    LOG.info("Available Resource: " + response.getAvailableResources());
    LOG.info("Num of Allocated Containers: " + response.getAllocatedContainers().size());
    if (response.getAllocatedContainers().size() > 0) {
        LOG.info("================================================================");
        for (Container container : response.getAllocatedContainers()) {
            LOG.info("> Container Id: " + container.getId());
            LOG.info("> Node Id: " + container.getNodeId());
            LOG.info("> Resource (Mem): " + container.getResource().getMemory());
            LOG.info("> State : " + container.getState());
            LOG.info("> Priority: " + container.getPriority());
        }/*  w  w  w  .  java  2  s .c o m*/
        LOG.info("================================================================");
    }

    Map<SubQueryId, List<Container>> allocated = new HashMap<SubQueryId, List<Container>>();
    if (allocatedContainers.size() > 0) {
        for (Container container : allocatedContainers) {
            SubQueryId subQueryId = subQueryMap.get(container.getPriority());
            SubQueryState state = context.getSubQuery(subQueryId).getState();
            if (!(isRunningState(state) && subQueryMap.containsKey(container.getPriority()))) {
                releaseAssignedContainer(container.getId());
                synchronized (subQueryMap) {
                    subQueryMap.remove(container.getPriority());
                }
            } else {
                if (allocated.containsKey(subQueryId)) {
                    allocated.get(subQueryId).add(container);
                } else {
                    allocated.put(subQueryId, Lists.newArrayList(container));
                }
            }
        }

        for (Entry<SubQueryId, List<Container>> entry : allocated.entrySet()) {
            eventHandler.handle(new SubQueryContainerAllocationEvent(entry.getKey(), entry.getValue()));
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• »