Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration() 

Source Link

Usage

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

@Test
public void validateFiniteContainerSelfShutdown() throws Exception {
    YarnApplication<Void> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerWithRandomDelay.class, ByteBuffer.wrap("hello".getBytes()))
            .containerCount(4).withApplicationMaster(new YarnConfiguration())
            .build("validateFiniteContainerSelfShutdown");
    yarnApplication.registerReplyListener(new ContainerReplyListener() {
        @Override/* ww  w  . j  av  a2s .  c  o m*/
        public void onReply(ByteBuffer replyData) {
            byte[] replyBytes = new byte[replyData.limit()];
            replyData.rewind();
            replyData.get(replyBytes);
            replyData.rewind();
            String reply = new String(replyBytes);
            System.out.println("REPLY: " + reply);
        }
    });
    yarnApplication.launch();
    assertTrue(yarnApplication.isRunning());
    yarnApplication.awaitFinish();
    assertFalse(yarnApplication.isRunning());
}

---

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

@Test
public void validateFiniteContainerSelfShutdownWithProcessException() throws Exception {
    //      ConfigUtils.addToClasspath(new File("/Users/oleg/HADOOP_DEV/yaya/yarn-test-cluster/src/main/resources"));
    YarnApplication<Void> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerThrowingExceptionRandomly.class,
                    ByteBuffer.wrap("hello".getBytes()))
            .containerCount(4).withApplicationMaster(new YarnConfiguration())
            .build("validateFiniteContainerSelfShutdownWithProcessException");
    yarnApplication.registerReplyListener(new ContainerReplyListener() {
        @Override/*  w  w  w  .j  a  v  a2s.co  m*/
        public void onReply(ByteBuffer replyData) {
            byte[] replyBytes = new byte[replyData.limit()];
            replyData.rewind();
            replyData.get(replyBytes);
            replyData.rewind();
            String reply = new String(replyBytes);
            System.out.println("REPLY: " + reply);
        }
    });
    yarnApplication.launch();
    assertTrue(yarnApplication.isRunning());
    yarnApplication.awaitFinish();
    assertFalse(yarnApplication.isRunning());
}

---

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

@Test
public void validateReusableContainerGracefulShutdownNoTaskSubmitted() throws Exception {
    //      ConfigUtils.addToClasspath(new File("/Users/oleg/HADOOP_DEV/yaya/yarn-test-cluster/src/main/resources"));
    YarnApplication<DataProcessor> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerWithRandomDelay.class).containerCount(4)
            .withApplicationMaster(new YarnConfiguration())
            .build("validateReusableContainerGracefulShutdownNoTaskSubmitted");
    yarnApplication.launch();/*from   w ww. ja va2  s. c  om*/
    assertTrue(yarnApplication.isRunning());
    yarnApplication.shutDown();
    assertFalse(yarnApplication.isRunning());
}

---

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

/**
 * Validates that application with long running containers can be shut down gracefully (via shutdown method)
 * after all currently submitted tasks have finished
 *///from   ww  w  . j av  a 2 s  .co m
@Test
public void validateReusableContainerGracefulShutdownAfterAllTasksSubmitted() throws Exception {
    //      ConfigUtils.addToClasspath(new File("/Users/oleg/HADOOP_DEV/yaya/yarn-test-cluster/src/main/resources"));
    YarnApplication<DataProcessor> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerWithRandomDelay.class).containerCount(4)
            .withApplicationMaster(new YarnConfiguration())
            .build("validateReusableContainerGracefulShutdownAfterAllTasksSubmitted");
    yarnApplication.registerReplyListener(new ContainerReplyListener() {
        @Override
        public void onReply(ByteBuffer replyData) {
            byte[] replyBytes = new byte[replyData.limit()];
            replyData.rewind();
            replyData.get(replyBytes);
            replyData.rewind();
            String reply = new String(replyBytes);
            System.out.println("REPLY: " + reply);
        }
    });
    DataProcessor dataProcessor = yarnApplication.launch();
    assertTrue(yarnApplication.isRunning());
    for (int i = 0; i < 10; i++) {
        dataProcessor.process(ByteBuffer.wrap(("hello-" + i).getBytes()));
    }
    yarnApplication.shutDown();
    assertFalse(yarnApplication.isRunning());
}

---

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

/**
 * Validates ability for long running containers with long running tasks to terminate.
 * In this test there should see no REPLY messages coming in.
 *//*from w ww  .  j  a v  a 2s  . c om*/
@Test
public void validateReusableContainerTermination() throws Exception {
    //      ConfigUtils.addToClasspath(new File("/Users/oleg/HADOOP_DEV/yaya/yarn-test-cluster/src/main/resources"));
    final YarnApplication<DataProcessor> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerWithLongRunningLoop.class).containerCount(4)
            .withApplicationMaster(new YarnConfiguration()).build("validateReusableContainerTermination");
    final AtomicBoolean replies = new AtomicBoolean();
    yarnApplication.registerReplyListener(new ContainerReplyListener() {
        @Override
        public void onReply(ByteBuffer replyData) {
            replies.set(true);
        }
    });
    final DataProcessor dataProcessor = yarnApplication.launch();
    assertTrue(yarnApplication.isRunning());
    ExecutorService executor = Executors.newCachedThreadPool();
    executor.execute(new Runnable() {
        @Override
        public void run() {
            for (int i = 0; i < 10; i++) {
                dataProcessor.process(ByteBuffer.wrap(("hello-" + i).getBytes()));
            }
        }
    });
    Thread.sleep(2000);// let it run for a while and then terminate
    executor.execute(new Runnable() {
        @Override
        public void run() {
            yarnApplication.shutDown();
        }
    });
    Thread.sleep(2000);
    assertTrue(yarnApplication.isRunning());
    yarnApplication.terminate();
    assertFalse(yarnApplication.isRunning());
    assertFalse(replies.get());
    executor.shutdownNow();
}

---

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

/**
 * Validates that a finite container with a long running task can be terminated
 *///  w w w .j ava2  s.  com
@Test
public void validateFiniteContainerTermination() throws Exception {
    //      ConfigUtils.addToClasspath(new File("/Users/oleg/HADOOP_DEV/yaya/yarn-test-cluster/src/main/resources"));
    final YarnApplication<Void> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerWithLongRunningLoop.class, ByteBuffer.wrap("hello".getBytes()))
            .containerCount(2).withApplicationMaster(new YarnConfiguration())
            .build("validateFiniteContainerTermination");
    yarnApplication.registerReplyListener(new ContainerReplyListener() {
        @Override
        public void onReply(ByteBuffer replyData) {
            byte[] replyBytes = new byte[replyData.limit()];
            replyData.rewind();
            replyData.get(replyBytes);
            replyData.rewind();
            String reply = new String(replyBytes);
            System.out.println("REPLY: " + reply);
        }
    });
    ExecutorService executor = Executors.newCachedThreadPool();
    executor.execute(new Runnable() {
        @Override
        public void run() {
            yarnApplication.launch();
            System.out.println("STARTED");
        }
    });
    yarnApplication.awaitLaunch();
    assertTrue(yarnApplication.isRunning());

    Thread.sleep(4000);// let it run for a while and then terminate
    executor.execute(new Runnable() {
        @Override
        public void run() {
            yarnApplication.shutDown();
        }
    });

    Thread.sleep(4000); // ensure that it is still running
    assertTrue(yarnApplication.isRunning());
    yarnApplication.terminate();
    assertFalse(yarnApplication.isRunning());
}

---

From source file:oz.hadoop.yarn.api.core.Clustered_ApplicationMasterTests.java

License:Apache License

/**
 * This test validates that finite application exits immediately upon container startup failure
 * /*ww  w . j  av a  2  s .  co m*/
 * @throws Exception
 */
@Test
public void validateApplicationSelfExitAfterContaierStartupFailure() throws Exception {
    //      ConfigUtils.addToClasspath(new File("/Users/oleg/HADOOP_DEV/yaya/yarn-test-cluster/src/main/resources"));

    YarnApplication<Void> yarnApplication = YarnAssembly
            .forApplicationContainer(ContainerWithLongRunningLoop.class, ByteBuffer.wrap("hello".getBytes()))
            .containerCount(8).withApplicationMaster(new YarnConfiguration())
            .build("validateApplicationSelfExitAfterContaierStartupFailure");

    Map<String, Object> specificationMap = new HashMap<>(yarnApplication.getApplicationSpecification());
    specificationMap.put("FORCE_CONTAINER_ERROR", true);
    specificationMap.put(YayaConstants.YARN_CONFIG, new YarnConfiguration());

    Constructor<?> launcherCtr = ReflectionUtils
            .getInvocableConstructor("oz.hadoop.yarn.api.core.ApplicationMasterLauncherImpl", Map.class);
    Object launcher = launcherCtr.newInstance(specificationMap);
    Method launchMethod = ReflectionUtils.getMethodAndMakeAccessible(launcher.getClass(), "launch");
    launchMethod.invoke(launcher);
    //Thread.sleep(2000);
    assertFalse(yarnApplication.isRunning());
}

---

From source file:oz.hadoop.yarn.api.YarnAssemblyTests.java

License:Apache License

@Test
public void validateCommandBasedContainerWithCustom() {
    YarnConfiguration yarnConfig = new YarnConfiguration();
    ApplicationMasterSpecBuilder<Void> applicationMasterBuilder = YarnAssembly
            .forApplicationContainer("ls -all").memory(28).containerCount(4).virtualCores(34).priority(3)
            .withApplicationMaster(yarnConfig);

    YarnApplication<Void> yarnApplication = applicationMasterBuilder.maxAttempts(12).priority(34)
            .queueName("foo").virtualCores(35).memory(1024).build("my-application");

    PrimitiveImmutableTypeMap specMap = (PrimitiveImmutableTypeMap) yarnApplication
            .getApplicationSpecification();
    assertEquals(35, specMap.getInt(YayaConstants.VIRTUAL_CORES));
    assertEquals(34, specMap.getInt(YayaConstants.PRIORITY));
    assertEquals(12, specMap.getInt(YayaConstants.MAX_ATTEMPTS));
    assertEquals("foo", specMap.getString(YayaConstants.QUEUE_NAME));
    assertEquals(1024, specMap.getInt(YayaConstants.MEMORY));
    assertFalse(specMap.getBoolean(YayaConstants.YARN_EMULATOR));
    assertEquals("my-application", specMap.getString(YayaConstants.APPLICATION_NAME));

    PrimitiveImmutableTypeMap containerSpecMap = (PrimitiveImmutableTypeMap) specMap
            .get(YayaConstants.CONTAINER_SPEC);
    assertEquals(4, containerSpecMap.getInt(YayaConstants.CONTAINER_COUNT));
    assertEquals(3, containerSpecMap.getInt(YayaConstants.PRIORITY));
    assertEquals(28, containerSpecMap.getInt(YayaConstants.MEMORY));
    assertEquals(34, containerSpecMap.getInt(YayaConstants.VIRTUAL_CORES));
    assertEquals("ls -all", containerSpecMap.getString(YayaConstants.COMMAND));
}

---

From source file:oz.hadoop.yarn.api.YarnAssemblyTests.java

License:Apache License

@Test
public void validateJavaBasedContainerWithCustom() {
    YarnConfiguration yarnConfig = new YarnConfiguration();
    ApplicationMasterSpecBuilder<DataProcessor> applicationMasterBuilder = YarnAssembly
            .forApplicationContainer(DslTestApplicationContainer.class).memory(28).containerCount(4)
            .virtualCores(34).priority(3).withApplicationMaster(yarnConfig);

    YarnApplication<DataProcessor> yarnApplication = applicationMasterBuilder.maxAttempts(12).priority(34)
            .queueName("foo").virtualCores(35).memory(1024).build("my-application");

    PrimitiveImmutableTypeMap specMap = (PrimitiveImmutableTypeMap) yarnApplication
            .getApplicationSpecification();
    assertEquals(35, specMap.getInt(YayaConstants.VIRTUAL_CORES));
    assertEquals(34, specMap.getInt(YayaConstants.PRIORITY));
    assertEquals(12, specMap.getInt(YayaConstants.MAX_ATTEMPTS));
    assertEquals("foo", specMap.getString(YayaConstants.QUEUE_NAME));
    assertEquals(1024, specMap.getInt(YayaConstants.MEMORY));
    assertFalse(specMap.getBoolean(YayaConstants.YARN_EMULATOR));
    assertEquals("my-application", specMap.getString(YayaConstants.APPLICATION_NAME));

    PrimitiveImmutableTypeMap containerSpecMap = (PrimitiveImmutableTypeMap) specMap
            .get(YayaConstants.CONTAINER_SPEC);
    assertEquals(4, containerSpecMap.getInt(YayaConstants.CONTAINER_COUNT));
    assertEquals(3, containerSpecMap.getInt(YayaConstants.PRIORITY));
    assertEquals(28, containerSpecMap.getInt(YayaConstants.MEMORY));
    assertEquals(34, containerSpecMap.getInt(YayaConstants.VIRTUAL_CORES));
    assertNull(containerSpecMap.get(YayaConstants.CONTAINER_ARG));
    assertEquals(DslTestApplicationContainer.class.getName(),
            containerSpecMap.getString(YayaConstants.CONTAINER_IMPL));
}

---

From source file:oz.hadoop.yarn.api.YarnAssemblyTests.java

License:Apache License

@Test
public void validateJavaBasedContainerWithArgumentsAndShell() {
    YarnConfiguration yarnConfig = new YarnConfiguration();
    ApplicationMasterSpecBuilder<Void> applicationMasterBuilder = YarnAssembly
            .forApplicationContainer(DslTestApplicationContainer.class, ByteBuffer.wrap("Hello".getBytes()),
                    "java")
            .memory(28).containerCount(4).virtualCores(34).priority(3).withApplicationMaster(yarnConfig);

    YarnApplication<Void> yarnApplication = applicationMasterBuilder.maxAttempts(12).priority(34)
            .queueName("foo").virtualCores(35).memory(1024).build("my-application");

    PrimitiveImmutableTypeMap specMap = (PrimitiveImmutableTypeMap) yarnApplication
            .getApplicationSpecification();
    assertEquals(35, specMap.getInt(YayaConstants.VIRTUAL_CORES));
    assertEquals(34, specMap.getInt(YayaConstants.PRIORITY));
    assertEquals(12, specMap.getInt(YayaConstants.MAX_ATTEMPTS));
    assertEquals("foo", specMap.getString(YayaConstants.QUEUE_NAME));
    assertEquals(1024, specMap.getInt(YayaConstants.MEMORY));
    assertFalse(specMap.getBoolean(YayaConstants.YARN_EMULATOR));
    assertEquals("my-application", specMap.getString(YayaConstants.APPLICATION_NAME));

    PrimitiveImmutableTypeMap containerSpecMap = (PrimitiveImmutableTypeMap) specMap
            .get(YayaConstants.CONTAINER_SPEC);
    assertEquals(4, containerSpecMap.getInt(YayaConstants.CONTAINER_COUNT));
    assertEquals(3, containerSpecMap.getInt(YayaConstants.PRIORITY));
    assertEquals(28, containerSpecMap.getInt(YayaConstants.MEMORY));
    assertEquals(34, containerSpecMap.getInt(YayaConstants.VIRTUAL_CORES));
    assertNotNull(containerSpecMap.get(YayaConstants.CONTAINER_ARG));
    assertEquals(DslTestApplicationContainer.class.getName(),
            containerSpecMap.getString(YayaConstants.CONTAINER_IMPL));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »