Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration() 

Source Link

Usage

From source file:org.apache.samza.job.yarn.refactor.YarnClusterResourceManager.java

License:Apache License

/**
 * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
 * @param config to instantiate the container manager with
 * @param jobModelManager the jobModel manager to get the job model (mostly for the UI)
 * @param callback the callback to receive events from Yarn.
 * @param samzaAppState samza app state for display in the UI
 *///w  w w. j av  a  2  s . c o m
public YarnClusterResourceManager(Config config, JobModelManager jobModelManager,
        ClusterResourceManager.Callback callback, SamzaAppState samzaAppState) {
    super(callback);
    hConfig = new YarnConfiguration();
    hConfig.set("fs.http.impl", HttpFileSystem.class.getName());

    MetricsRegistryMap registry = new MetricsRegistryMap();

    // parse configs from the Yarn environment
    String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    String nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
    String nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
    String nodeHttpPortString = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());

    int nodePort = Integer.parseInt(nodePortString);
    int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
    YarnConfig yarnConfig = new YarnConfig(config);
    int interval = yarnConfig.getAMPollIntervalMs();

    //Instantiate the AM Client.
    this.amClient = AMRMClientAsync.createAMRMClientAsync(interval, this);

    this.state = new YarnAppState(jobModelManager, -1, containerId, nodeHostString, nodePort, nodeHttpPort,
            samzaAppState);

    log.info("Initialized YarnAppState: {}", state.toString());
    this.service = new SamzaAppMasterService(config, this.state, registry);

    log.info("ContainerID str {}, Nodehost  {} , Nodeport  {} , NodeHttpport {}",
            new Object[] { containerIdStr, nodeHostString, nodePort, nodeHttpPort });
    this.lifecycle = new SamzaYarnAppMasterLifecycle(yarnConfig.getContainerMaxMemoryMb(),
            yarnConfig.getContainerMaxCpuCores(), state, amClient);

    yarnContainerRunner = new YarnContainerRunner(config, hConfig);
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapSuccess() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");

    configMap.put("yarn.resources.myResource2.path", "https://host2.com/package");
    configMap.put("yarn.resources.myResource2.local.name", "__package");
    configMap.put("yarn.resources.myResource2.local.type", "archive");
    configMap.put("yarn.resources.myResource2.local.visibility", "private");

    configMap.put("yarn.resources.myResource3.path", "https://host3.com/csr");
    configMap.put("yarn.resources.myResource3.local.name", "csr");
    configMap.put("yarn.resources.myResource3.local.type", "file");
    configMap.put("yarn.resources.myResource3.local.visibility", "application");

    configMap.put("otherconfig", "https://host4.com/not_included");
    configMap.put("yarn.resources.myResource4.local.name", "notExisting");
    configMap.put("yarn.resources.myResource4.local.type", "file");
    configMap.put("yarn.resources.myResource4.local.visibility", "application");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);// w  w w. j  a va2s  . c  o m
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertEquals("resourceMap has 3 resources", 3, resourceMap.size());

    // resource1
    assertEquals("host1.com", resourceMap.get("readme").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("readme").getType());
    assertEquals(LocalResourceVisibility.PUBLIC, resourceMap.get("readme").getVisibility());

    // resource 2
    assertEquals("host2.com", resourceMap.get("__package").getResource().getHost());
    assertEquals(LocalResourceType.ARCHIVE, resourceMap.get("__package").getType());
    assertEquals(LocalResourceVisibility.PRIVATE, resourceMap.get("__package").getVisibility());

    // resource 3
    assertEquals("host3.com", resourceMap.get("csr").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("csr").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("csr").getVisibility());

    // resource 4 should not exist
    assertNull("Resource does not exist with the name myResource4", resourceMap.get("myResource4"));
    assertNull("Resource does not exist with the defined config name notExisting for myResource4 either",
            resourceMap.get("notExisting"));
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithDefaultValues() {

    Map<String, String> configMap = new HashMap<>();

    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);// ww w .j  av a  2 s  .  c o  m
    Map<String, LocalResource> resourceMap = mapper.getResourceMap();

    assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
    assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
    assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
    assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
    assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithFileStatusFailure() {
    thrown.expect(LocalizerResourceException.class);
    thrown.expectMessage("IO Exception when accessing the resource file status from the filesystem");

    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "unknown://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");
    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);/*from w ww .  j  av a2  s.com*/
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithInvalidVisibilityFailure() {
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(/*w  ww  .jav a  2 s . co  m*/
            "No enum constant org.apache.hadoop.yarn.api.records.LocalResourceVisibility.INVALIDVISIBILITY");

    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "file");
    configMap.put("yarn.resources.myResource1.local.visibility", "invalidVisibility");
    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);
}

---

From source file:org.apache.samza.job.yarn.TestLocalizerResourceMapper.java

License:Apache License

@Test
public void testResourceMapWithInvalidTypeFailure() {
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage("No enum constant org.apache.hadoop.yarn.api.records.LocalResourceType.INVALIDTYPE");

    Map<String, String> configMap = new HashMap<>();
    configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");
    configMap.put("yarn.resources.myResource1.local.name", "readme");
    configMap.put("yarn.resources.myResource1.local.type", "invalidType");
    configMap.put("yarn.resources.myResource1.local.visibility", "public");
    Config conf = new MapConfig(configMap);

    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());
    yarnConfiguration.set("fs.https.impl", HttpFileSystem.class.getName());
    LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf),
            yarnConfiguration);//from  w w w  .  j  a  v  a 2s  . co  m
}

---

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

@Test
public void testSamzaTaskManager() throws Exception {
    Map<String, String> conf = new HashMap<>();
    conf.putAll(getConfig());/*from  ww  w.j  a  v a2s .  c o  m*/
    conf.put("yarn.container.memory.mb", "500");
    conf.put("yarn.container.cpu.cores", "5");

    SamzaTaskManager taskManager = new SamzaTaskManager(new MapConfig(conf), state, amRmClientAsync,
            new YarnConfiguration());

    AbstractContainerAllocator allocator = (AbstractContainerAllocator) getPrivateFieldFromTaskManager(
            "containerAllocator", taskManager).get(taskManager);
    assertEquals(ContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMaxMemoryMb);
    assertEquals(5, allocator.containerMaxCpuCore);

    conf.clear();
    conf.putAll(getConfigWithHostAffinity());
    conf.put("yarn.container.memory.mb", "500");
    conf.put("yarn.container.cpu.cores", "5");

    taskManager = new SamzaTaskManager(new MapConfig(conf), state, amRmClientAsync, new YarnConfiguration());

    allocator = (AbstractContainerAllocator) getPrivateFieldFromTaskManager("containerAllocator", taskManager)
            .get(taskManager);
    assertEquals(HostAwareContainerAllocator.class, allocator.getClass());
    // Asserts that samza exposed container configs is honored by allocator thread
    assertEquals(500, allocator.containerMaxMemoryMb);
    assertEquals(5, allocator.containerMaxCpuCore);
}

---

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

@Test
public void testOnInit() throws Exception {
    Config conf = getConfig();//  w  w w . ja v a  2s  .co m
    SamzaTaskManager taskManager = new SamzaTaskManager(conf, state, amRmClientAsync, new YarnConfiguration());

    MockContainerAllocator allocator = new MockContainerAllocator(amRmClientAsync,
            TestUtil.getContainerUtil(getConfig(), state), new YarnConfig(conf));
    getPrivateFieldFromTaskManager("containerAllocator", taskManager).set(taskManager, allocator);

    getPrivateFieldFromTaskManager("allocatorThread", taskManager).set(taskManager, new Thread() {
        public void run() {
            isRunning = true;
        }
    });

    taskManager.onInit();
    Thread.sleep(1000);

    // Verify Allocator thread has started running
    assertTrue(isRunning);

    // Verify the remaining state
    assertEquals(1, state.neededContainers.get());
    assertEquals(1, allocator.requestedContainers);

    taskManager.onShutdown();
}

---

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

@Test
public void testOnShutdown() throws Exception {
    SamzaTaskManager taskManager = new SamzaTaskManager(getConfig(), state, amRmClientAsync,
            new YarnConfiguration());
    taskManager.onInit();/*from   w ww .ja v a  2 s  .  c  om*/

    Thread.sleep(100);

    Thread allocatorThread = (Thread) getPrivateFieldFromTaskManager("allocatorThread", taskManager)
            .get(taskManager);
    assertTrue(allocatorThread.isAlive());

    taskManager.onShutdown();

    Thread.sleep(100);
    assertFalse(allocatorThread.isAlive());

}

---

From source file:org.apache.samza.job.yarn.TestSamzaTaskManager.java

License:Apache License

/**
 * Test Task Manager should stop when all containers finish
 *//*from  w ww.  java 2 s . c  o m*/
@Test
public void testTaskManagerShouldStopWhenContainersFinish() {
    SamzaTaskManager taskManager = new SamzaTaskManager(getConfig(), state, amRmClientAsync,
            new YarnConfiguration());

    taskManager.onInit();

    assertFalse(taskManager.shouldShutdown());

    taskManager.onContainerCompleted(
            TestUtil.getContainerStatus(state.amContainerId, ContainerExitStatus.SUCCESS, ""));

    assertTrue(taskManager.shouldShutdown());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »