Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration() 

Source Link

Usage

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

/**
 * Test Task Manager should request a new container when a task fails with unknown exit code
 * When host-affinity is not enabled, it will always request for ANY_HOST
 *//*  w w  w.  j  av  a2 s  . com*/
@Test
public void testNewContainerRequestedOnFailureWithUnknownCode() throws Exception {
    Config conf = getConfig();
    SamzaTaskManager taskManager = new SamzaTaskManager(conf, state, amRmClientAsync, new YarnConfiguration());
    MockContainerAllocator allocator = new MockContainerAllocator(amRmClientAsync,
            TestUtil.getContainerUtil(getConfig(), state), new YarnConfig(conf));
    getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);

    Thread thread = new Thread(allocator);
    getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);

    // onInit triggers a request
    taskManager.onInit();

    assertFalse(taskManager.shouldShutdown());
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());

    Container container = TestUtil
            .getContainer(ConverterUtils.toContainerId("container_1350670447861_0003_01_000002"), "abc", 123);
    taskManager.onContainerAllocated(container);

    // Allow container to run and update state
    Thread.sleep(300);

    // Create first container failure
    taskManager.onContainerCompleted(
            TestUtil.getContainerStatus(container.getId(), 1, "Expecting a failure here"));

    // The above failure should trigger a container request
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());
    assertEquals(ContainerRequestState.ANY_HOST,
            allocator.containerRequestState.getRequestsQueue().peek().getPreferredHost());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(2, testAMRMClient.requests.size());
    assertEquals(0, testAMRMClient.getRelease().size());

    taskManager.onContainerAllocated(container);

    // Allow container to run and update state
    Thread.sleep(300);

    assertTrue(state.jobHealthy.get());

    // Create a second failure
    taskManager.onContainerCompleted(
            TestUtil.getContainerStatus(container.getId(), 1, "Expecting a failure here"));

    // The above failure should trigger a job shutdown because our retry count is set to 1
    assertEquals(0, allocator.containerRequestState.getRequestsQueue().size());
    assertEquals(2, testAMRMClient.requests.size());
    assertEquals(0, testAMRMClient.getRelease().size());
    assertFalse(state.jobHealthy.get());
    assertTrue(taskManager.shouldShutdown());
    assertEquals(FinalApplicationStatus.FAILED, state.status);

    taskManager.onShutdown();
}

---

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

/**
 * Test Task Manager should request a new container when a task fails with unknown exit code
 * When host-affinity is enabled, it will always request for the same host that it was last seen on
 *///from   w ww  . j av  a2 s .  co m
@Test
public void testSameContainerRequestedOnFailureWithUnknownCode() throws Exception {
    Config conf = getConfigWithHostAffinity();
    SamzaTaskManager taskManager = new SamzaTaskManager(conf, state, amRmClientAsync, new YarnConfiguration());
    MockContainerAllocator allocator = new MockContainerAllocator(amRmClientAsync,
            TestUtil.getContainerUtil(getConfig(), state), new YarnConfig(conf));
    getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);

    Thread thread = new Thread(allocator);
    getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);

    // onInit triggers a request
    taskManager.onInit();

    assertFalse(taskManager.shouldShutdown());
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());

    Container container = TestUtil
            .getContainer(ConverterUtils.toContainerId("container_1350670447861_0003_01_000002"), "abc", 123);
    taskManager.onContainerAllocated(container);

    // Allow container to run and update state
    Thread.sleep(300);

    // Create first container failure
    taskManager.onContainerCompleted(
            TestUtil.getContainerStatus(container.getId(), 1, "Expecting a failure here"));

    // The above failure should trigger a container request
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());
    assertEquals("abc", allocator.containerRequestState.getRequestsQueue().peek().getPreferredHost());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(2, testAMRMClient.requests.size());
    assertEquals(0, testAMRMClient.getRelease().size());

    taskManager.onContainerAllocated(container);

    // Allow container to run and update state
    Thread.sleep(300);

    assertTrue(state.jobHealthy.get());

    // Create a second failure
    taskManager.onContainerCompleted(
            TestUtil.getContainerStatus(container.getId(), 1, "Expecting a failure here"));

    // The above failure should trigger a job shutdown because our retry count is set to 1
    assertEquals(0, allocator.containerRequestState.getRequestsQueue().size());
    assertEquals(2, testAMRMClient.requests.size());
    assertEquals(0, testAMRMClient.getRelease().size());
    assertFalse(state.jobHealthy.get());
    assertTrue(taskManager.shouldShutdown());
    assertEquals(FinalApplicationStatus.FAILED, state.status);

    taskManager.onShutdown();
}

---

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

/**
 * Test AM requests a new container when a task fails
 * Error codes with same behavior - Disk failure, preemption and aborted
 */// w  ww  .  jav  a 2s  .  co  m
@Test
public void testNewContainerRequestedOnFailureWithKnownCode() throws Exception {
    Map<String, String> config = new HashMap<>();
    config.putAll(getConfig());
    config.remove("yarn.container.retry.count");

    SamzaTaskManager taskManager = new SamzaTaskManager(new MapConfig(config), state, amRmClientAsync,
            new YarnConfiguration());
    MockContainerAllocator allocator = new MockContainerAllocator(amRmClientAsync,
            TestUtil.getContainerUtil(getConfig(), state), new YarnConfig(new MapConfig(config)));
    getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);

    Thread thread = new Thread(allocator);
    getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, thread);

    // Start the task manager
    taskManager.onInit();
    assertFalse(taskManager.shouldShutdown());
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());

    Container container = TestUtil
            .getContainer(ConverterUtils.toContainerId("container_1350670447861_0003_01_000002"), "abc", 123);
    taskManager.onContainerAllocated(container);

    // Allow container to run and update state
    Thread.sleep(300);

    // Create container failure - with ContainerExitStatus.DISKS_FAILED
    taskManager.onContainerCompleted(
            TestUtil.getContainerStatus(container.getId(), ContainerExitStatus.DISKS_FAILED, "Disk failure"));

    // The above failure should trigger a container request
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(2, testAMRMClient.requests.size());
    assertEquals(0, testAMRMClient.getRelease().size());
    assertEquals(ContainerRequestState.ANY_HOST,
            allocator.containerRequestState.getRequestsQueue().peek().getPreferredHost());

    // Create container failure - with ContainerExitStatus.PREEMPTED
    taskManager.onContainerCompleted(TestUtil.getContainerStatus(container.getId(),
            ContainerExitStatus.PREEMPTED, "Task Preempted by RM"));

    // The above failure should trigger a container request
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(ContainerRequestState.ANY_HOST,
            allocator.containerRequestState.getRequestsQueue().peek().getPreferredHost());

    // Create container failure - with ContainerExitStatus.ABORTED
    taskManager.onContainerCompleted(TestUtil.getContainerStatus(container.getId(), ContainerExitStatus.ABORTED,
            "Task Aborted by the NM"));

    // The above failure should trigger a container request
    assertEquals(1, allocator.containerRequestState.getRequestsQueue().size());
    assertEquals(2, testAMRMClient.requests.size());
    assertEquals(0, testAMRMClient.getRelease().size());
    assertFalse(taskManager.shouldShutdown());
    assertFalse(state.jobHealthy.get());
    assertEquals(ContainerRequestState.ANY_HOST,
            allocator.containerRequestState.getRequestsQueue().peek().getPreferredHost());

    taskManager.onShutdown();
}

---

From source file:org.apache.samza.job.yarn.util.TestUtil.java

License:Apache License

/**
 * Returns MockContainerUtil instance with a Mock NMClient
 * *///from  w  w  w. j  ava2 s.c  o m
public static ContainerUtil getContainerUtil(Config config, SamzaAppState state) {
    return new MockContainerUtil(config, state, new YarnConfiguration(), new MockNMClient("Mock NMClient"));
}

---

From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java

License:Apache License

/**
 * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
 * @param config to instantiate the cluster manager with
 * @param jobModelManager the jobModel manager to get the job model (mostly for the UI)
 * @param callback the callback to receive events from Yarn.
 * @param samzaAppState samza app state for display in the UI
 *//*  w w w .  j  a  va2  s  . com*/
public YarnClusterResourceManager(Config config, JobModelManager jobModelManager,
        ClusterResourceManager.Callback callback, SamzaApplicationState samzaAppState) {
    super(callback);
    yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

    // Use the Samza job config "fs.<scheme>.impl" and "fs.<scheme>.impl.*" for YarnConfiguration
    FileSystemImplConfig fsImplConfig = new FileSystemImplConfig(config);
    fsImplConfig.getSchemes().forEach(scheme -> {
        fsImplConfig.getSchemeConfig(scheme)
                .forEach((confKey, confValue) -> yarnConfiguration.set(confKey, confValue));
    });

    MetricsRegistryMap registry = new MetricsRegistryMap();
    metrics = new SamzaAppMasterMetrics(config, samzaAppState, registry, getClass().getClassLoader());

    // parse configs from the Yarn environment
    String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    String nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
    String nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
    String nodeHttpPortString = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());

    int nodePort = Integer.parseInt(nodePortString);
    int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
    YarnConfig yarnConfig = new YarnConfig(config);
    this.yarnConfig = yarnConfig;
    this.config = config;
    int interval = yarnConfig.getAMPollIntervalMs();

    //Instantiate the AM Client.
    this.amClient = AMRMClientAsync.createAMRMClientAsync(interval, this);

    this.state = new YarnAppState(-1, containerId, nodeHostString, nodePort, nodeHttpPort);

    log.info("Initialized YarnAppState: {}", state.toString());
    this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry,
            yarnConfiguration);

    log.info("Container ID: {}, Nodehost:  {} , Nodeport : {} , NodeHttpport: {}", containerIdStr,
            nodeHostString, nodePort, nodeHttpPort);
    ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
    this.lifecycle = new SamzaYarnAppMasterLifecycle(clusterManagerConfig.getContainerMemoryMb(),
            clusterManagerConfig.getNumCores(), samzaAppState, state, amClient);
    this.nmClientAsync = NMClientAsync.createNMClientAsync(this);

}

---

From source file:org.apache.samza.validation.YarnJobValidationTool.java

License:Apache License

public static void main(String[] args) throws Exception {
    CommandLine cmdline = new CommandLine();
    OptionParser parser = cmdline.parser();
    OptionSpec<String> validatorOpt = parser.accepts("metrics-validator", "The metrics validator class.")
            .withOptionalArg().ofType(String.class).describedAs("com.foo.bar.ClassName");
    OptionSet options = cmdline.parser().parse(args);
    Config config = cmdline.loadConfig(options);
    MetricsValidator validator = null;//from  w w w. ja  va  2s.  c o  m
    if (options.has(validatorOpt)) {
        String validatorClass = options.valueOf(validatorOpt);
        validator = Util.getObj(validatorClass, MetricsValidator.class);
    }

    YarnConfiguration hadoopConfig = new YarnConfiguration();
    hadoopConfig.set("fs.http.impl", HttpFileSystem.class.getName());
    hadoopConfig.set("fs.https.impl", HttpFileSystem.class.getName());
    ClientHelper clientHelper = new ClientHelper(hadoopConfig);

    new YarnJobValidationTool(new JobConfig(config), clientHelper.yarnClient(), validator).run();
}

---

From source file:org.apache.slider.client.SliderClient.java

License:Apache License

/**
 * Constructor//from  w w  w.  j a  v  a2 s  .  c  om
 */
public SliderClient() {
    super("Slider Client");
    new HdfsConfiguration();
    new YarnConfiguration();
}

---

From source file:org.apache.slider.providers.slideram.SliderAMProviderService.java

License:Apache License

@Override
public void applyInitialRegistryDefinitions(URL amWebURI, URL agentOpsURI, URL agentStatusURI,
        ServiceRecord serviceRecord) throws IOException {
    super.applyInitialRegistryDefinitions(amWebURI, agentOpsURI, agentStatusURI, serviceRecord);
    // now publish site.xml files
    YarnConfiguration defaultYarnConfig = new YarnConfiguration();
    amState.getPublishedSliderConfigurations().put(PublishedArtifacts.COMPLETE_CONFIG,
            new PublishedConfiguration("Complete slider application settings", getConfig(), getConfig()));
    amState.getPublishedSliderConfigurations().put(PublishedArtifacts.YARN_SITE_CONFIG,
            new PublishedConfiguration("YARN site settings", ConfigHelper.loadFromResource("yarn-site.xml"),
                    defaultYarnConfig));

    amState.getPublishedSliderConfigurations().put(PublishedArtifacts.CORE_SITE_CONFIG,
            new PublishedConfiguration("Core site settings", ConfigHelper.loadFromResource("core-site.xml"),
                    defaultYarnConfig));
    amState.getPublishedSliderConfigurations().put(PublishedArtifacts.HDFS_SITE_CONFIG,
            new PublishedConfiguration("HDFS site settings", ConfigHelper.loadFromResource("hdfs-site.xml"),
                    new HdfsConfiguration(true)));

    try {//from w  ww .ja va  2  s  .  c o  m

        URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
        URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY);

        URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);

        // Set the configurations URL.

        String configurationsURL = SliderUtils.appendToURL(publisherURL.toExternalForm(),
                RestPaths.SLIDER_CONFIGSET);
        String exportsURL = SliderUtils.appendToURL(publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS);

        serviceRecord.addExternalEndpoint(
                RegistryTypeUtils.webEndpoint(CustomRegistryConstants.WEB_UI, amWebURI.toURI()));
        serviceRecord.addExternalEndpoint(RegistryTypeUtils
                .restEndpoint(CustomRegistryConstants.MANAGEMENT_REST_API, managementAPI.toURI()));
        serviceRecord.addExternalEndpoint(RegistryTypeUtils
                .restEndpoint(CustomRegistryConstants.PUBLISHER_REST_API, publisherURL.toURI()));
        serviceRecord.addExternalEndpoint(RegistryTypeUtils
                .restEndpoint(CustomRegistryConstants.REGISTRY_REST_API, registryREST.toURI()));
        serviceRecord.addExternalEndpoint(RegistryTypeUtils.restEndpoint(
                CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, new URI(configurationsURL)));
        serviceRecord.addExternalEndpoint(RegistryTypeUtils
                .restEndpoint(CustomRegistryConstants.PUBLISHER_EXPORTS_API, new URI(exportsURL)));

    } catch (URISyntaxException e) {
        throw new IOException(e);
    }
}

---

From source file:org.apache.slider.server.appmaster.SliderAppMaster.java

License:Apache License

/**
 * Service Constructor/*from   ww  w  . j  av a  2 s.com*/
 */
public SliderAppMaster() {
    super(SERVICE_CLASSNAME_SHORT);
    new HdfsConfiguration();
    new YarnConfiguration();
}

---

From source file:org.apache.slider.server.services.utility.AbstractSliderLaunchedService.java

License:Apache License

public AbstractSliderLaunchedService(String name) {
    super(name);//from   w  w  w  .ja  v  a 2s.  c o  m
    // make sure all the yarn configs get loaded
    YarnConfiguration conf = new YarnConfiguration();
    ConfigHelper.registerDeprecatedConfigItems();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »