Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APP_CONTAINER_LOG

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APP_CONTAINER_LOG_DIR.

Prototype

String YARN_APP_CONTAINER_LOG_DIR

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APP_CONTAINER_LOG_DIR.

Click Source Link

Document

The log directory for the containers

Usage

From source file:org.apache.tez.client.TestTezClientUtils.java

License:Apache License

@Test(timeout = 5000)
public void testAMLoggingOptsSimple() throws IOException, YarnException {

    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");

    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    DAG dag = DAG.create("testdag");
    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
            .setTaskLaunchCmdOpts("initialLaunchOpts"));
    AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(),
            new Credentials());
    ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId,
            dag, "amName", amConf, new HashMap<String, LocalResource>(), new Credentials(), false,
            new TezApiVersionInfo(), mock(HistoryACLPolicyManager.class));

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
    assertEquals(1, commands.size());//from www .j a va2 s  .  c o m
    for (String expectedCmd : expectedCommands) {
        assertTrue(commands.get(0).contains(expectedCmd));
    }

    Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
    String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
    assertNull(logEnv);
}

From source file:org.apache.tez.client.TestTezClientUtils.java

License:Apache License

@Test(timeout = 5000)
public void testAMLoggingOptsPerLogger() throws IOException, YarnException {

    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
            "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");

    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    DAG dag = DAG.create("testdag");
    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
            .setTaskLaunchCmdOpts("initialLaunchOpts"));
    AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(),
            new Credentials());
    ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId,
            dag, "amName", amConf, new HashMap<String, LocalResource>(), new Credentials(), false,
            new TezApiVersionInfo(), mock(HistoryACLPolicyManager.class));

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
    assertEquals(1, commands.size());/* ww  w .  j a v  a2 s.  co m*/
    for (String expectedCmd : expectedCommands) {
        assertTrue(commands.get(0).contains(expectedCmd));
    }

    Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
    String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
    assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv);
}

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

@Private
@VisibleForTesting// w  w  w  . java 2s . c o  m
public static void addLog4jSystemProperties(String logLevel, List<String> vargs) {
    vargs.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    vargs.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    vargs.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + logLevel + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
}

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskLoggingOptsPerLogger() {

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL,
            "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }/*from  w ww  .j a v  a  2 s  . co  m*/

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
    }
}

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskLoggingOptsSimple() {

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG");

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }/*from   ww w  . j  a  va 2 s. co m*/

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }
}

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskSpecificLoggingOpts() {

    String vertexName = "testvertex";
    String customJavaOpts = "-Xmx128m";

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    // Expected command opts for regular tasks
    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "INFO" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 3; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }/* w  w w .  ja v  a 2  s.c o m*/

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }

    // Expected command opts for instrumented tasks.
    expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < 3; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.tez=INFO", logEnvVal);
    }
}

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskSpecificLoggingOpts2() {

    String vertexName = "testvertex";
    String customJavaOpts = "-Xmx128m";

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "WARN;org.apache.tez=INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    // Expected command opts for regular tasks
    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 3; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }//  w w  w .j  a  va 2 s  .  c  o  m

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.tez=INFO", logEnvVal);
    }

    // Expected command opts for instrumented tasks.
    expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < 3; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }
}

From source file:org.testifyproject.resource.yarn.MiniYarnResource.java

License:Apache License

@Override
public YarnConfiguration configure(TestContext testContext, LocalResource localResource,
        PropertiesReader configReader) {
    String testName = testContext.getName();
    String logDirectory = fileSystemUtil.createPath("target", "yarn", testName);

    YarnConfiguration configuration = new YarnConfiguration();
    configuration.set(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR, logDirectory);
    configuration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    configuration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);

    return configuration;
}

From source file:org.testifyproject.resource.yarn.MiniYarnResource.java

License:Apache License

@Override
public LocalResourceInstance<MiniYARNCluster, YarnClient> start(TestContext testContext,
        LocalResource localResource, YarnConfiguration config) throws Exception {
    String logDirectory = config.get(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
    fileSystemUtil.recreateDirectory(logDirectory);
    server = new MiniYARNCluster(testContext.getName(), 1, 1, 1, 1, true);
    server.init(config);/*  w  w w.j  av a  2  s. co  m*/
    server.start();

    client = YarnClient.createYarnClient();
    client.init(server.getConfig());
    client.start();

    return LocalResourceInstanceBuilder.builder().resource(server).client(client).build("yarn", localResource);
}

Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_APP_CONTAINER_LOG_DIR

Introduction

Prototype

Document

Usage