Example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_CONTAINER_TEMP_DIR

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_CONTAINER_TEMP_DIR.

Prototype

String DEFAULT_CONTAINER_TEMP_DIR

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration DEFAULT_CONTAINER_TEMP_DIR.

Click Source Link

Document

Container temp directory

Usage

From source file:com.datatorrent.stram.LaunchContainerRunnable.java

License:Apache License

/**
 * Build the command to launch the child VM in the container
 *
 * @param jvmID/*  w w  w .j  a v  a 2 s  .c  o m*/
 * @return
 */
public List<CharSequence> getChildVMCommand(String jvmID) {

    List<CharSequence> vargs = new ArrayList<CharSequence>(8);

    if (!StringUtils.isBlank(System.getenv(Environment.JAVA_HOME.key()))) {
        // node manager provides JAVA_HOME
        vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    } else {
        vargs.add("java");
    }

    String jvmOpts = dag.getAttributes().get(LogicalPlan.CONTAINER_JVM_OPTIONS);
    if (jvmOpts == null) {
        if (dag.isDebug()) {
            vargs.add(JAVA_REMOTE_DEBUG_OPTS);
        }
    } else {
        Map<String, String> params = new HashMap<String, String>();
        params.put("applicationId",
                Integer.toString(container.getId().getApplicationAttemptId().getApplicationId().getId()));
        params.put("containerId", Integer.toString(container.getId().getId()));
        StrSubstitutor sub = new StrSubstitutor(params, "%(", ")");
        vargs.add(sub.replace(jvmOpts));
        if (dag.isDebug() && !jvmOpts.contains("-agentlib:jdwp=")) {
            vargs.add(JAVA_REMOTE_DEBUG_OPTS);
        }
    }

    List<DAG.OperatorMeta> operatorMetaList = Lists.newArrayList();
    int bufferServerMemory = 0;
    for (PTOperator operator : sca.getContainer().getOperators()) {
        bufferServerMemory += operator.getBufferServerMemory();
        operatorMetaList.add(operator.getOperatorMeta());
    }
    Context.ContainerOptConfigurator containerOptConfigurator = dag.getAttributes()
            .get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR);
    jvmOpts = containerOptConfigurator.getJVMOptions(operatorMetaList);
    jvmOpts = parseJvmOpts(jvmOpts, ((long) bufferServerMemory) * MB_TO_B);
    LOG.info("Jvm opts {} for container {}", jvmOpts, container.getId());
    vargs.add(jvmOpts);

    Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
    vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
    vargs.add("-Djava.io.tmpdir=" + childTmpDir);
    vargs.add(String.format("-D%scid=%s", StreamingApplication.DT_PREFIX, jvmID));
    vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
    vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);

    String loggersLevel = System.getProperty(DTLoggerFactory.DT_LOGGERS_LEVEL);
    if (loggersLevel != null) {
        vargs.add(String.format("-D%s=%s", DTLoggerFactory.DT_LOGGERS_LEVEL, loggersLevel));
    }
    // Add main class and its arguments
    vargs.add(StreamingContainer.class.getName()); // main of Child

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

    // Final commmand
    StringBuilder mergedCommand = new StringBuilder(256);
    for (CharSequence str : vargs) {
        mergedCommand.append(str).append(" ");
    }
    List<CharSequence> vargsFinal = new ArrayList<CharSequence>(1);
    vargsFinal.add(mergedCommand.toString());
    return vargsFinal;

}

---

From source file:org.apache.tez.dag.utils.TezEngineChildJVM.java

License:Apache License

public static List<String> getVMCommand(InetSocketAddress taskAttemptListenerAddr, TezConfiguration conf,
        String containerIdentifier, String tokenIdentifier, int applicationAttemptNumber, boolean shouldProfile,
        String javaOpts) {/*from   www  .  j a v a  2  s  . c o  m*/

    Vector<String> vargs = new Vector<String>(9);

    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");

    //set custom javaOpts
    vargs.add(javaOpts);

    //[Debug Task] Current simplest way to attach debugger to  Tez Child Task
    // Uncomment the following, then launch a regular job
    // Works best on one-box configured with a single container (hence one task at a time). 
    //    LOG.error(" !!!!!!!!! Launching Child-Task in debug/suspend mode.  Attach to port 8003 !!!!!!!!");
    //    vargs.add("-Xdebug -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8003,server=y,suspend=y");

    Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
    vargs.add("-Djava.io.tmpdir=" + childTmpDir);

    // FIXME Setup the log4j properties

    // Decision to profile needs to be made in the scheduler.
    if (shouldProfile) {
        // FIXME add support for profiling
    }

    // Add main class and its arguments 
    vargs.add(YarnTezDagChild.class.getName()); // main of Child

    // pass TaskAttemptListener's address
    vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress());
    vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
    vargs.add(containerIdentifier);
    vargs.add(tokenIdentifier);
    vargs.add(Integer.toString(applicationAttemptNumber));

    vargs.add("1>" + getTaskLogFile(LogName.STDOUT));
    vargs.add("2>" + getTaskLogFile(LogName.STDERR));

    // Final commmand
    StringBuilder mergedCommand = new StringBuilder();
    for (CharSequence str : vargs) {
        mergedCommand.append(str).append(" ");
    }
    Vector<String> vargsFinal = new Vector<String>(1);
    vargsFinal.add(mergedCommand.toString());
    return vargsFinal;
}

---

From source file:org.apache.tez.dag.utils.TezRuntimeChildJVM.java

License:Apache License

public static List<String> getVMCommand(InetSocketAddress taskAttemptListenerAddr, String containerIdentifier,
        String tokenIdentifier, int applicationAttemptNumber, String javaOpts) {

    Vector<String> vargs = new Vector<String>(9);

    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");

    //set custom javaOpts
    vargs.add(javaOpts);/*from   www  .jav  a  2 s  .c  o  m*/

    Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
    vargs.add("-Djava.io.tmpdir=" + childTmpDir);

    // Add main class and its arguments
    vargs.add(TezChild.class.getName()); // main of Child

    // pass TaskAttemptListener's address
    vargs.add(taskAttemptListenerAddr.getAddress().getHostAddress());
    vargs.add(Integer.toString(taskAttemptListenerAddr.getPort()));
    vargs.add(containerIdentifier);
    vargs.add(tokenIdentifier);
    vargs.add(Integer.toString(applicationAttemptNumber));

    vargs.add("1>" + getTaskLogFile(LogName.STDOUT));
    vargs.add("2>" + getTaskLogFile(LogName.STDERR));

    // TODO Is this StringBuilder really required ? YARN already accepts a list of commands.
    // Final commmand
    StringBuilder mergedCommand = new StringBuilder();
    for (CharSequence str : vargs) {
        mergedCommand.append(str).append(" ");
    }
    Vector<String> vargsFinal = new Vector<String>(1);
    vargsFinal.add(mergedCommand.toString());
    return vargsFinal;
}

---

From source file:oz.hadoop.yarn.test.cluster.InJvmContainerExecutor.java

License:Apache License

/**
 * Most of this code is copied from the super class's launchContainer method (unfortunately), since directory
 * and other preparation logic is tightly coupled with the actual container launch.
 * Would be nice if it was broken apart where launch method would be invoked when
 * everything is prepared/*from ww  w.  j  av  a  2s  .  co  m*/
 */
private void prepareContainerDirectories(Container container, Path nmPrivateContainerScriptPath,
        Path nmPrivateTokensPath, String userName, String appId, Path containerWorkDir, List<String> localDirs,
        List<String> logDirs) {

    FsPermission dirPerm = new FsPermission(APPDIR_PERM);
    ContainerId containerId = container.getContainerId();
    String containerIdStr = ConverterUtils.toString(containerId);
    String appIdStr = ConverterUtils.toString(containerId.getApplicationAttemptId().getApplicationId());

    try {
        for (String sLocalDir : localDirs) {
            Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
            Path userdir = new Path(usersdir, userName);
            Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE);
            Path appDir = new Path(appCacheDir, appIdStr);
            Path containerDir = new Path(appDir, containerIdStr);
            createDir(containerDir, dirPerm, true);
        }

        // Create the container log-dirs on all disks
        this.createLogDirs(appIdStr, containerIdStr, logDirs);

        Path tmpDir = new Path(containerWorkDir, YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
        createDir(tmpDir, dirPerm, false);

        // copy launch script to work dir
        Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
        fc.util().copy(nmPrivateContainerScriptPath, launchDst);

        // copy container tokens to work dir
        Path tokenDst = new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
        fc.util().copy(nmPrivateTokensPath, tokenDst);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to prepare container directories for container " + container,
                e);
    }
}

---