Example usage for org.apache.hadoop.yarn.client.api YarnClient getContainers

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api YarnClient getContainers.

Prototype

public abstract List<ContainerReport> getContainers(ApplicationAttemptId applicationAttemptId)
        throws YarnException, IOException;

Source Link

Document

Get a report of all (Containers) of ApplicationAttempt in the cluster.

Usage

From source file:io.hops.hopsworks.common.admin.llap.LlapClusterFacade.java

License:Open Source License

public List<String> getLlapHosts() {
    ArrayList<String> hosts = new ArrayList<>();

    if (!isClusterUp() || isClusterStarting()) {
        return hosts;
    }/*from  w w  w  . j  a  va2 s . co  m*/

    // The cluster is app, so the appId exists
    String llapAppID = variablesFacade.getVariableValue(Settings.VARIABLE_LLAP_APP_ID);

    ApplicationId appId = ApplicationId.fromString(llapAppID);
    YarnClient yarnClient = yarnClientService.getYarnClientSuper(settings.getConfiguration()).getYarnClient();
    try {
        List<ApplicationAttemptReport> attempts = yarnClient.getApplicationAttempts(appId);
        ApplicationAttemptReport current = null;
        for (ApplicationAttemptReport attempt : attempts) {
            // Only if the app is running the metrics are available
            if (attempt.getYarnApplicationAttemptState() == YarnApplicationAttemptState.RUNNING) {
                current = attempt;
                break;
            }
        }

        if (current == null) {
            return hosts;
        }

        List<ContainerReport> containerReports = yarnClient.getContainers(current.getApplicationAttemptId());

        // For all the new/running containers, which are not the application master, get the host
        for (ContainerReport containerReport : containerReports) {
            // Only if the container is running the metrics are available
            if (containerReport.getContainerState() == ContainerState.RUNNING
                    && !containerReport.getContainerId().equals(current.getAMContainerId())) {
                hosts.add(containerReport.getAssignedNode().getHost());
            }
        }

    } catch (IOException | YarnException ex) {
        logger.log(Level.SEVERE, "Couldn't retrieve the containers for LLAP cluster", ex);
    } finally {
        try {
            yarnClient.close();
        } catch (IOException ex) {
        }
    }

    return hosts;
}

---

From source file:org.apache.flink.yarn.YarnConfigurationITCase.java

License:Apache License

/**
 * Tests that the Flink components are started with the correct
 * memory settings.// w  ww .  jav  a 2s  . co  m
 */
@Test(timeout = 60000)
public void testFlinkContainerMemory() throws Exception {
    final YarnClient yarnClient = getYarnClient();
    final Configuration configuration = new Configuration(flinkConfiguration);

    final int masterMemory = 64;
    final int taskManagerMemory = 128;
    final int slotsPerTaskManager = 3;

    // disable heap cutoff min
    configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);
    configuration.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(1L << 20));
    configuration.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(4L << 20));

    final YarnConfiguration yarnConfiguration = getYarnConfiguration();
    final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(configuration, yarnConfiguration,
            CliFrontend.getConfigurationDirectoryFromEnv(), yarnClient, true);

    clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
    clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));

    final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");

    final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile);
    final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1);

    try {
        final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
                .setMasterMemoryMB(masterMemory).setTaskManagerMemoryMB(taskManagerMemory)
                .setSlotsPerTaskManager(slotsPerTaskManager).createClusterSpecification();

        final ClusterClient<ApplicationId> clusterClient = clusterDescriptor
                .deployJobCluster(clusterSpecification, jobGraph, true);

        final ApplicationId clusterId = clusterClient.getClusterId();

        final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration),
                TestingUtils.defaultExecutor());

        try {
            final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);

            final ApplicationAttemptId currentApplicationAttemptId = applicationReport
                    .getCurrentApplicationAttemptId();

            // wait until we have second container allocated
            List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);

            while (containers.size() < 2) {
                // this is nasty but Yarn does not offer a better way to wait
                Thread.sleep(50L);
                containers = yarnClient.getContainers(currentApplicationAttemptId);
            }

            for (ContainerReport container : containers) {
                if (container.getContainerId().getId() == 1) {
                    // this should be the application master
                    assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
                } else {
                    assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
                }
            }

            final URI webURI = new URI(clusterClient.getWebInterfaceURL());

            CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
            Collection<TaskManagerInfo> taskManagerInfos;

            while (true) {
                taskManagersInfoCompletableFuture = restClient.sendRequest(webURI.getHost(), webURI.getPort(),
                        TaskManagersHeaders.getInstance(), EmptyMessageParameters.getInstance(),
                        EmptyRequestBody.getInstance());

                final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();

                taskManagerInfos = taskManagersInfo.getTaskManagerInfos();

                // wait until the task manager has registered and reported its slots
                if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
                    break;
                } else {
                    Thread.sleep(100L);
                }
            }

            // there should be at least one TaskManagerInfo
            final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();

            assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));

            final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters
                    .create(configuration, taskManagerMemory, slotsPerTaskManager);

            final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;

            // We compare here physical memory assigned to a container with the heap memory that we should pass to
            // jvm as Xmx parameter. Those value might differ significantly due to sytem page size or jvm
            // implementation therefore we use 15% threshold here.
            assertThat((double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap()
                    / (double) expectedHeadSize, is(closeTo(1.0, 0.15)));
        } finally {
            restClient.shutdown(TIMEOUT);
            clusterClient.shutdown();
        }

        clusterDescriptor.killCluster(clusterId);

    } finally {
        clusterDescriptor.close();
    }
}

---

From source file:probos.TestProblem.java

License:Open Source License

void dotest(int N, String defn) throws Exception {
    if (System.getenv("HADOOP_HOME") == null && System.getenv("HADOOP_COMMON_HOME") == null)
        fail("HADOOP_HOME must be set");
    if (System.getenv("JAVA_HOME") == null)
        fail("JAVA_HOME must be set");
    File luaFile = File.createTempFile("kittenFile", ".lua");
    FileWriter w = new FileWriter(luaFile);
    w.write(defn);/*from   ww  w  .  j  ava 2  s.  c  om*/
    w.close();
    Map<String, Object> extraLuaValues = ImmutableMap.<String, Object>of();
    Map<String, String> extraLocalResources = ImmutableMap.<String, String>of();

    YarnClientParameters params = new LuaYarnClientParameters(luaFile.toString(), "probos", yConf,
            extraLuaValues, extraLocalResources);
    YarnClientService service = new YarnClientServiceImpl(params);
    service.startAndWait();
    while (!service.isApplicationFinished()) {
        Thread.sleep(1000);
    }
    assertEquals(FinalApplicationStatus.SUCCEEDED, service.getFinalReport().getFinalApplicationStatus());

    ApplicationAttemptId aaid = service.getFinalReport().getCurrentApplicationAttemptId();
    YarnClient yc = new YarnClientFactory(this.yConf).connect();
    List<ContainerReport> lcr = yc.getContainers(aaid);
    for (ContainerReport cr : lcr) {
        String stdErrURL = "http:" + cr.getLogUrl() + "/stderr?start=0";
        System.err.println(cr.getContainerId().toString() + " " + stdErrURL);
        String stderr = getURL(stdErrURL);
        System.err.println(stderr);
        assertFalse("Container" + cr.getContainerId().toString() + " " + stderr,
                stderr.contains("ArrayIndexOutOfBoundsException"));
    }

    //service.getFinalReport().get

    System.err.println();
    Thread.sleep(60000);
    for (int id = 1; id <= N; id++)
        for (String type : new String[] { "o", "e" }) {
            String file = HERE + "/testHostname." + type + "1-" + id;
            assertTrue("File not found " + file, new File(file).exists());
        }
}

---