Example usage for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR.

Prototype

String LOG_DIR_EXPANSION_VAR

To view the source code for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR.

Click Source Link

Document

The temporary environmental variable for container log directory.

Usage

From source file:org.apache.tez.client.TestTezClientUtils.java

License:Apache License

@Test(timeout = 5000)
public void testAMLoggingOptsSimple() throws IOException, YarnException {

    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN");

    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    DAG dag = DAG.create("testdag");
    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
            .setTaskLaunchCmdOpts("initialLaunchOpts"));
    AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(),
            new Credentials());
    ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId,
            dag, "amName", amConf, new HashMap<String, LocalResource>(), new Credentials(), false,
            new TezApiVersionInfo(), mock(HistoryACLPolicyManager.class));

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
    assertEquals(1, commands.size());/*  w w  w.  ja v a  2s .co m*/
    for (String expectedCmd : expectedCommands) {
        assertTrue(commands.get(0).contains(expectedCmd));
    }

    Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
    String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
    assertNull(logEnv);
}

---

From source file:org.apache.tez.client.TestTezClientUtils.java

License:Apache License

@Test(timeout = 5000)
public void testAMLoggingOptsPerLogger() throws IOException, YarnException {

    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
            "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");

    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    DAG dag = DAG.create("testdag");
    dag.addVertex(Vertex.create("testVertex", ProcessorDescriptor.create("processorClassname"), 1)
            .setTaskLaunchCmdOpts("initialLaunchOpts"));
    AMConfiguration amConf = new AMConfiguration(tezConf, new HashMap<String, LocalResource>(),
            new Credentials());
    ApplicationSubmissionContext appSubmissionContext = TezClientUtils.createApplicationSubmissionContext(appId,
            dag, "amName", amConf, new HashMap<String, LocalResource>(), new Credentials(), false,
            new TezApiVersionInfo(), mock(HistoryACLPolicyManager.class));

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    List<String> commands = appSubmissionContext.getAMContainerSpec().getCommands();
    assertEquals(1, commands.size());//from w  w w  .  j  av  a2  s.  c o m
    for (String expectedCmd : expectedCommands) {
        assertTrue(commands.get(0).contains(expectedCmd));
    }

    Map<String, String> environment = appSubmissionContext.getAMContainerSpec().getEnvironment();
    String logEnv = environment.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
    assertEquals("org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG", logEnv);
}

---

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

/**
 * Create an ApplicationSubmissionContext to launch a Tez AM
 * @param appId Application Id//from  w  w  w .j a  v a2s .co m
 * @param dag DAG to be submitted
 * @param amName Name for the application
 * @param amConfig AM Configuration
 * @param tezJarResources Resources to be used by the AM
 * @param sessionCreds the credential object which will be populated with session specific
 * @param historyACLPolicyManager
 * @return an ApplicationSubmissionContext to launch a Tez AM
 * @throws IOException
 * @throws YarnException
 */
@Private
@VisibleForTesting
public static ApplicationSubmissionContext createApplicationSubmissionContext(ApplicationId appId, DAG dag,
        String amName, AMConfiguration amConfig, Map<String, LocalResource> tezJarResources,
        Credentials sessionCreds, boolean tezLrsAsArchive, TezApiVersionInfo apiVersionInfo,
        HistoryACLPolicyManager historyACLPolicyManager) throws IOException, YarnException {

    Preconditions.checkNotNull(sessionCreds);
    TezConfiguration conf = amConfig.getTezConfiguration();

    FileSystem fs = TezClientUtils.ensureStagingDirExists(conf, TezCommonUtils.getTezBaseStagingPath(conf));
    String strAppId = appId.toString();
    Path tezSysStagingPath = TezCommonUtils.createTezSystemStagingPath(conf, strAppId);
    Path binaryConfPath = TezCommonUtils.getTezConfStagingPath(tezSysStagingPath);
    binaryConfPath = fs.makeQualified(binaryConfPath);

    // Setup resource requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB,
            TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT));
    capability.setVirtualCores(amConfig.getTezConfiguration().getInt(
            TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT));
    if (LOG.isDebugEnabled()) {
        LOG.debug("AppMaster capability = " + capability);
    }

    // Setup required Credentials for the AM launch. DAG specific credentials
    // are handled separately.
    ByteBuffer securityTokens = null;
    // Setup security tokens
    Credentials amLaunchCredentials = new Credentials();
    if (amConfig.getCredentials() != null) {
        amLaunchCredentials.addAll(amConfig.getCredentials());
    }

    // Add Staging dir creds to the list of session credentials.
    TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { binaryConfPath }, conf);

    // Add session specific credentials to the AM credentials.
    amLaunchCredentials.mergeAll(sessionCreds);

    DataOutputBuffer dob = new DataOutputBuffer();
    amLaunchCredentials.writeTokenStorageToStream(dob);
    securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

    // Setup the command to run the AM
    List<String> vargs = new ArrayList<String>(8);
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");

    String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability);
    vargs.add(amOpts);

    String amLogLevelString = amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_LOG_LEVEL,
            TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
    String[] amLogParams = parseLogParams(amLogLevelString);

    String amLogLevel = amLogParams[0];
    maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);

    // FIX sun bug mentioned in TEZ-327
    vargs.add("-Dsun.nio.ch.bugLevel=''");

    vargs.add(TezConstants.TEZ_APPLICATION_MASTER_CLASS);
    if (dag == null) {
        vargs.add("--" + TezConstants.TEZ_SESSION_MODE_CLI_OPTION);
    }

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDOUT);
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR);

    Vector<String> vargsFinal = new Vector<String>(8);
    // Final command
    StringBuilder mergedCommand = new StringBuilder();
    for (CharSequence str : vargs) {
        mergedCommand.append(str).append(" ");
    }
    vargsFinal.add(mergedCommand.toString());

    if (LOG.isDebugEnabled()) {
        LOG.debug("Command to launch container for ApplicationMaster is : " + mergedCommand);
    }

    Map<String, String> environment = new TreeMap<String, String>();
    TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV,
            TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive);

    addVersionInfoToEnv(environment, apiVersionInfo);
    addLogParamsToEnv(environment, amLogParams);

    Map<String, LocalResource> amLocalResources = new TreeMap<String, LocalResource>();

    // Not fetching credentials for AMLocalResources. Expect this to be provided via AMCredentials.
    if (amConfig.getAMLocalResources() != null) {
        amLocalResources.putAll(amConfig.getAMLocalResources());
    }
    amLocalResources.putAll(tezJarResources);

    // Setup Session ACLs and update conf as needed
    Map<String, String> aclConfigs = null;
    if (historyACLPolicyManager != null) {
        if (dag == null) {
            aclConfigs = historyACLPolicyManager.setupSessionACLs(amConfig.getTezConfiguration(), appId);
        } else {
            // Non-session mode
            // As only a single DAG is support, we should combine AM and DAG ACLs under the same
            // acl management layer
            aclConfigs = historyACLPolicyManager.setupNonSessionACLs(amConfig.getTezConfiguration(), appId,
                    dag.getDagAccessControls());
        }
    }

    // emit conf as PB file
    ConfigurationProto finalConfProto = createFinalConfProtoForApp(amConfig.getTezConfiguration(), aclConfigs);

    FSDataOutputStream amConfPBOutBinaryStream = null;
    try {
        amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath);
        finalConfProto.writeTo(amConfPBOutBinaryStream);
    } finally {
        if (amConfPBOutBinaryStream != null) {
            amConfPBOutBinaryStream.close();
        }
    }

    LocalResource binaryConfLRsrc = TezClientUtils.createLocalResource(fs, binaryConfPath,
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION);
    amConfig.setBinaryConfLR(binaryConfLRsrc);
    amLocalResources.put(TezConstants.TEZ_PB_BINARY_CONF_NAME, binaryConfLRsrc);

    // Create Session Jars definition to be sent to AM as a local resource
    Path sessionJarsPath = TezCommonUtils.getTezAMJarStagingPath(tezSysStagingPath);
    FSDataOutputStream sessionJarsPBOutStream = null;
    try {
        sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, sessionJarsPath);
        // Write out the initial list of resources which will be available in the AM
        DAGProtos.PlanLocalResourcesProto amResourceProto;
        if (amLocalResources != null && !amLocalResources.isEmpty()) {
            amResourceProto = DagTypeConverters.convertFromLocalResources(amLocalResources);
        } else {
            amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance();
        }
        amResourceProto.writeDelimitedTo(sessionJarsPBOutStream);
    } finally {
        if (sessionJarsPBOutStream != null) {
            sessionJarsPBOutStream.close();
        }
    }

    LocalResource sessionJarsPBLRsrc = TezClientUtils.createLocalResource(fs, sessionJarsPath,
            LocalResourceType.FILE, LocalResourceVisibility.APPLICATION);
    amLocalResources.put(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, sessionJarsPBLRsrc);

    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration());
    Map<ApplicationAccessType, String> acls = aclManager.toYARNACls();

    if (dag != null) {

        DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, sessionCreds);

        // emit protobuf DAG file style
        Path binaryPath = TezCommonUtils.getTezBinPlanStagingPath(tezSysStagingPath);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Stage directory information for AppId :" + appId + " tezSysStagingPath :"
                    + tezSysStagingPath + " binaryConfPath :" + binaryConfPath + " sessionJarsPath :"
                    + sessionJarsPath + " binaryPlanPath :" + binaryPath);
        }

        FSDataOutputStream dagPBOutBinaryStream = null;

        try {
            //binary output
            dagPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryPath);
            dagPB.writeTo(dagPBOutBinaryStream);
        } finally {
            if (dagPBOutBinaryStream != null) {
                dagPBOutBinaryStream.close();
            }
        }

        amLocalResources.put(TezConstants.TEZ_PB_PLAN_BINARY_NAME, TezClientUtils.createLocalResource(fs,
                binaryPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));

        if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) {
            Path textPath = localizeDagPlanAsText(dagPB, fs, amConfig, strAppId, tezSysStagingPath);
            amLocalResources.put(TezConstants.TEZ_PB_PLAN_TEXT_NAME, TezClientUtils.createLocalResource(fs,
                    textPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
        }
    }

    // Setup ContainerLaunchContext for AM container
    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(amLocalResources, environment,
            vargsFinal, null, securityTokens, acls);

    // Set up the ApplicationSubmissionContext
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE);
    appContext.setApplicationId(appId);
    appContext.setResource(capability);
    if (amConfig.getQueueName() != null) {
        appContext.setQueue(amConfig.getQueueName());
    }
    appContext.setApplicationName(amName);
    appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean(
            TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION,
            TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT));
    appContext.setAMContainerSpec(amContainer);

    appContext.setMaxAppAttempts(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
            TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT));

    return appContext;

}

---

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

@Private
@VisibleForTesting/*from  ww  w . j a v a2s . c  om*/
public static void addLog4jSystemProperties(String logLevel, List<String> vargs) {
    vargs.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    vargs.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    vargs.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + logLevel + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskLoggingOptsPerLogger() {

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL,
            "DEBUG;org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO");

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }/*  w  w  w. ja  va  2 s  . c o m*/

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.hadoop.ipc=INFO;org.apache.hadoop.server=INFO", logEnvVal);
    }
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskLoggingOptsSimple() {

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "DEBUG");

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();
        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }/*  w  w w .  j  a  v a2  s. com*/

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskSpecificLoggingOpts() {

    String vertexName = "testvertex";
    String customJavaOpts = "-Xmx128m";

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG;org.apache.tez=INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    // Expected command opts for regular tasks
    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "INFO" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 3; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }/*from www. j  a v  a2s .  co m*/

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }

    // Expected command opts for instrumented tasks.
    expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < 3; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.tez=INFO", logEnvVal);
    }
}

---

From source file:org.apache.tez.dag.app.dag.impl.TestVertexImpl2.java

License:Apache License

@Test(timeout = 5000)
public void testTaskSpecificLoggingOpts2() {

    String vertexName = "testvertex";
    String customJavaOpts = "-Xmx128m";

    Configuration conf = new TezConfiguration();
    conf.set(TezConfiguration.TEZ_TASK_LOG_LEVEL, "WARN;org.apache.tez=INFO");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, vertexName + "[0,1,2]");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL, "DEBUG");
    conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, customJavaOpts);

    LogTestInfoHolder testInfo = new LogTestInfoHolder(conf);

    // Expected command opts for regular tasks
    List<String> expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 3; i < testInfo.numTasks; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }//from w w w. j av  a2 s  . c om

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertEquals("org.apache.tez=INFO", logEnvVal);
    }

    // Expected command opts for instrumented tasks.
    expectedCommands = new LinkedList<String>();
    expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
    expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
    expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + ","
            + TezConstants.TEZ_CONTAINER_LOGGER_NAME);

    for (int i = 0; i < 3; i++) {
        ContainerContext containerContext = testInfo.vertex.getContainerContext(i);
        String javaOpts = containerContext.getJavaOpts();

        assertTrue(javaOpts.contains(testInfo.initialJavaOpts));
        for (String expectedCmd : expectedCommands) {
            assertTrue(javaOpts.contains(expectedCmd));
        }

        Map<String, String> env = containerContext.getEnvironment();
        String val = env.get(testInfo.envKey);
        assertEquals(testInfo.envVal, val);
        String logEnvVal = env.get(TezConstants.TEZ_CONTAINER_LOG_PARAMS);
        assertNull(logEnvVal);
    }
}

---

From source file:org.apache.tez.dag.utils.TezEngineChildJVM.java

License:Apache License

private static String getTaskLogFile(LogName filter) {
    return ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + filter.toString();
}

---

From source file:org.chenchun.ApplicationMaster.java

License:Apache License

public static void main(String[] args) throws IOException, YarnException, InterruptedException {
    final String params = args[0];
    final int containerNum = Integer.valueOf(args[1]);

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<AMRMClient.ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);// w w  w  .  j  a v a 2 s  .c o  m
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    System.out.println("registerApplicationMaster 0");
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("registerApplicationMaster 1");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i = 0; i < containerNum; ++i) {
        AMRMClient.ContainerRequest containerAsk = new AMRMClient.ContainerRequest(capability, null, null,
                priority);
        System.out.println("Making res-req " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    while (completedContainers < containerNum) {
        AllocateResponse response = rmClient.allocate(responseId++);
        System.out.println("Allocate response " + response.getAMCommand() + " " + "allocate "
                + response.getAllocatedContainers().size() + "contains");
        for (Container container : response.getAllocatedContainers()) {
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(params + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                            + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));
            System.out.println("Launching container " + container.getId() + " on " + container.getNodeId());
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            System.out.println("Completed container " + status.getContainerId());
        }
        Thread.sleep(1000);
    }
    System.out.println("Unregister ApplicationMaster");

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• »