Example usage for org.apache.hadoop.yarn.api ApplicationConstants STDERR

List of usage examples for org.apache.hadoop.yarn.api ApplicationConstants STDERR

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.*
  5. ApplicationConstants
  6. STDERR

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api ApplicationConstants STDERR.

Prototype

String STDERR

To view the source code for org.apache.hadoop.yarn.api ApplicationConstants STDERR.

Click Source Link

Usage

From source file:com.github.hdl.tensorflow.yarn.app.TFContainer.java

License:Apache License

public StringBuilder makeCommands(long containerMemory, String clusterSpec, String jobName, int taskIndex) {
    // Set the necessary command to execute on the allocated container
    Vector<CharSequence> vargs = new Vector<CharSequence>(5);
    vargs.add(ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java");
    //vargs.add("-Xmx" + containerMemory + "m");
    vargs.add("-Xmx" + containerMemory + "m");
    String containerClassName = TFServerLauncher.class.getName();
    vargs.add(containerClassName);// w w w  .  j  a va 2s  . c om
    vargs.add("--" + TFServerLauncher.OPT_CS + " " + clusterSpec);
    vargs.add("--" + TFServerLauncher.OPT_JN + " " + jobName);
    vargs.add("--" + TFServerLauncher.OPT_TI + " " + taskIndex);
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerLauncher."
            + ApplicationConstants.STDOUT);
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerLauncher."
            + ApplicationConstants.STDERR);

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    return command;
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private String buildApplicationMasterCommand(int memoryMbs) {
    String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
    return new StringBuilder().append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
            .append(" -Xmx").append(memoryMbs).append("M").append(" ")
            .append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs)).append(" ")
            .append(GobblinApplicationMaster.class.getName()).append(" --")
            .append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME).append(" ")
            .append(this.applicationName).append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
            .append(File.separator).append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
            .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator)
            .append(appMasterClassName).append(".").append(ApplicationConstants.STDERR).toString();
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException {
    FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem());
    rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration());

    LogCopier.Builder builder = LogCopier.newBuilder().useSrcFileSystem(this.fs).useDestFileSystem(rawLocalFs)
            .readFrom(getHdfsLogDir(appWorkDir)).writeTo(sinkLogDir).acceptsLogFileExtensions(
                    ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR));
    if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
        builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
    }/* ww  w. j a  va2  s  . co m*/
    if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
        builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
    }
    return builder.build();
}

From source file:gobblin.yarn.GobblinYarnLogSource.java

License:Apache License

/**
 * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}.
 *
 * @param config the {@link Config} use to create the {@link LogCopier}
 * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in
 * @param destFs the destination {@link FileSystem}
 * @param appWorkDir the Gobblin Yarn application working directory on HDFS
 * @return a {@link LogCopier} instance/*from   w  w  w .ja  va2  s.c o  m*/
 * @throws IOException if it fails on any IO operation
 */
protected LogCopier buildLogCopier(Config config, ContainerId containerId, FileSystem destFs, Path appWorkDir)
        throws IOException {
    LogCopier.Builder builder = LogCopier.newBuilder()
            .useSrcFileSystem(FileSystem.getLocal(new Configuration())).useDestFileSystem(destFs)
            .readFrom(getLocalLogDir()).writeTo(getHdfsLogDir(containerId, destFs, appWorkDir))
            .acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR))
            .useLogFileNamePrefix(containerId.toString());
    if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE)) {
        builder.useMaxBytesPerLogFile(config.getBytes(GobblinYarnConfigurationKeys.LOG_COPIER_MAX_FILE_SIZE));
    }
    if (config.hasPath(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER)) {
        builder.useScheduler(config.getString(GobblinYarnConfigurationKeys.LOG_COPIER_SCHEDULER));
    }
    return builder.build();
}

From source file:gobblin.yarn.YarnService.java

License:Apache License

private String buildContainerCommand(Container container, String helixInstanceName) {
    String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName();
    return new StringBuilder().append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
            .append(" -Xmx").append(container.getResource().getMemory()).append("M").append(" ")
            .append(JvmUtils.formatJvmArguments(this.containerJvmArgs)).append(" ")
            .append(GobblinYarnTaskRunner.class.getName()).append(" --")
            .append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME).append(" ")
            .append(this.applicationName).append(" --")
            .append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME).append(" ")
            .append(helixInstanceName).append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
            .append(File.separator).append(containerProcessName).append(".").append(ApplicationConstants.STDOUT)
            .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator)
            .append(containerProcessName).append(".").append(ApplicationConstants.STDERR).toString();
}

From source file:ml.shifu.guagua.yarn.util.YarnUtils.java

License:Apache License

private static StringBuilder getCommandBase(String mainClass, String vmArgs, String programArgs,
        String memory) {// w  w w. jav  a 2 s. c  o m
    List<String> commands = new ArrayList<String>(8);
    commands.add("exec");
    commands.add(Environment.JAVA_HOME.$() + File.separator + "bin" + File.separator + "java");
    commands.add("-Xms" + memory + "m");
    commands.add("-Xmx" + memory + "m");
    if (vmArgs != null) {
        commands.add(vmArgs);
    }
    commands.add("-cp .:${CLASSPATH}");
    commands.add(mainClass);
    if (programArgs != null) {
        commands.add(programArgs);
    }
    commands.add(
            "1> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDOUT);
    commands.add(
            "2> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR);

    StringBuilder sb = new StringBuilder(200);
    for (String cmd : commands) {
        sb.append(cmd).append(" ");
    }
    return sb;
}

From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

@VisibleForTesting
protected String buildApplicationMasterCommand(int memoryMbs) {
    String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
    return new StringBuilder().append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
            .append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) - this.jvmMemoryOverheadMbs)
            .append("M").append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=")
            .append(this.containerTimezone).append(" -D")
            .append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=")
            .append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(" -D")
            .append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=")
            .append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT).append(" ")
            .append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs)).append(" ")
            .append(GobblinApplicationMaster.class.getName()).append(" --")
            .append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME).append(" ")
            .append(this.applicationName).append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
            .append(File.separator).append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
            .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator)
            .append(appMasterClassName).append(".").append(ApplicationConstants.STDERR).toString();
}

From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException {
    FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem());
    rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration());

    LogCopier.Builder builder = LogCopier.newBuilder().useSrcFileSystem(this.fs).useDestFileSystem(rawLocalFs)
            .readFrom(getHdfsLogDir(appWorkDir)).writeTo(sinkLogDir).acceptsLogFileExtensions(
                    ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR));
    return builder.build();
}

From source file:org.apache.gobblin.yarn.YarnService.java

License:Apache License

@VisibleForTesting
protected String buildContainerCommand(Container container, String helixInstanceName) {
    String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName();
    return new StringBuilder().append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
            .append(" -Xmx")
            .append((int) (container.getResource().getMemory() * this.jvmMemoryXmxRatio)
                    - this.jvmMemoryOverheadMbs)
            .append("M").append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=")
            .append(this.containerTimezone).append(" -D")
            .append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=")
            .append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(" -D")
            .append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=")
            .append(containerProcessName).append(".").append(ApplicationConstants.STDOUT).append(" ")
            .append(JvmUtils.formatJvmArguments(this.containerJvmArgs)).append(" ")
            .append(GobblinYarnTaskRunner.class.getName()).append(" --")
            .append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME).append(" ")
            .append(this.applicationName).append(" --")
            .append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME).append(" ")
            .append(helixInstanceName).append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
            .append(File.separator).append(containerProcessName).append(".").append(ApplicationConstants.STDOUT)
            .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator)
            .append(containerProcessName).append(".").append(ApplicationConstants.STDERR).toString();
}

From source file:org.apache.samza.job.yarn.ContainerUtil.java

License:Apache License

public void runContainer(int samzaContainerId, Container container) {
    String containerIdStr = ConverterUtils.toString(container.getId());
    log.info("Got available container ID ({}) for container: {}", samzaContainerId, container);

    String cmdBuilderClassName;/*  w w w .  j  av a2s  . c o  m*/
    if (taskConfig.getCommandClass().isDefined()) {
        cmdBuilderClassName = taskConfig.getCommandClass().get();
    } else {
        cmdBuilderClassName = ShellCommandBuilder.class.getName();
    }
    CommandBuilder cmdBuilder = (CommandBuilder) Util.getObj(cmdBuilderClassName);
    cmdBuilder.setConfig(config).setId(samzaContainerId).setUrl(state.coordinatorUrl);

    String command = cmdBuilder.buildCommand();
    log.info("Container ID {} using command {}", samzaContainerId, command);

    log.info("Container ID {} using environment variables: ", samzaContainerId);
    Map<String, String> env = new HashMap<String, String>();
    for (Map.Entry<String, String> entry : cmdBuilder.buildEnvironment().entrySet()) {
        String escapedValue = Util.envVarEscape(entry.getValue());
        env.put(entry.getKey(), escapedValue);
        log.info("{}={} ", entry.getKey(), escapedValue);
    }

    Path path = new Path(yarnConfig.getPackagePath());
    log.info("Starting container ID {} using package path {}", samzaContainerId, path);

    startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
            command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR));

    if (state.neededContainers.decrementAndGet() == 0) {
        state.jobHealthy.set(true);
    }
    state.runningContainers.put(samzaContainerId, new YarnContainer(container));

    log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).",
            new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(),
                    container.getNodeHttpAddress(), containerIdStr });

    log.info("Started container ID {}", samzaContainerId);
}