Example usage for org.apache.hadoop.yarn.conf YarnConfiguration getStrings

List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration getStrings

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.conf.*
  5. YarnConfiguration
  6. getStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration getStrings.

Prototype

public String[] getStrings(String name, String... defaultValue)

Source Link

Document

Get the comma delimited values of the name property as an array of Strings.

Usage

From source file:com.gpiskas.yarn.Utils.java

License:Open Source License

public static void setUpEnv(Map<String, String> env, YarnConfiguration conf) {
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$())
            .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
        classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
        classPathEnv.append(c.trim());/*w w w. j  av a 2  s. c  o m*/
    }
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties");

    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }
    env.put("CLASSPATH", classPathEnv.toString());
}

From source file:com.hazelcast.yarn.HazelcastYarnClient.java

License:Open Source License

private void setEnv(Map<String, String> envs, YarnConfiguration conf) {
    for (String property : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(envs, Environment.CLASSPATH.name(), property.trim(), File.pathSeparator);
    }//from  w ww.j a v  a2s .  co  m

    Apps.addToEnvironment(envs, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*",
            File.pathSeparator);
}

From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java

License:Open Source License

/**
 * /*from  www.j  a  va 2s  . c o  m*/
 * @param yconf
 * @return
 * @throws IOException
 */
private Map<String, String> constructEnvionmentMap(YarnConfiguration yconf) throws IOException {
    Map<String, String> eMap = new HashMap<String, String>();

    //setup default app master environment
    StringBuilder classpath = new StringBuilder();
    for (String value : yconf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(value.trim());
    }

    //setup mapreduce appmaster environment (for robustness if not included in default environment)
    //for example, by default HDP 2.2 did not include mapred client libraries in this configuration
    //note: we cannot use mapreduce.application.classpath because it refers to HDFS and $PWD that needs to be setup       
    Map<String, String> env = System.getenv();
    String mapred_home = null;
    //get mapred home via alternative environment variables
    if (env.containsKey(MAPRED_HOME_ENV_CONST)) {
        mapred_home = env.get(MAPRED_HOME_ENV_CONST);
    } else if (env.containsKey(HADOOP_HOME_ENV_CONST)) {
        String tmp = env.get(HADOOP_HOME_ENV_CONST);
        mapred_home = tmp + File.separator + ".." + File.separator + "hadoop-mapreduce";
    }
    //concatenate mapred home libs to classpath
    if (mapred_home != null) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "*");
        classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "lib" + File.separator + "*");
    }

    eMap.put(Environment.CLASSPATH.name(), classpath.toString());
    MRApps.setClasspath(eMap, yconf);

    LOG.debug("Constructed environment classpath: " + classpath.toString());

    return eMap;
}

From source file:org.apache.helix.provisioning.yarn.YarnProvisioner.java

License:Apache License

private ContainerLaunchContext createLaunchContext(ContainerId containerId, Container container,
        Participant participant) throws Exception {

    ContainerLaunchContext participantContainer = Records.newRecord(ContainerLaunchContext.class);

    // Map<String, String> envs = System.getenv();
    String appName = applicationMasterConfig.getAppName();
    int appId = applicationMasterConfig.getAppId();
    String serviceName = _resourceConfig.getId().stringify();
    String serviceClasspath = applicationMasterConfig.getClassPath(serviceName);
    String mainClass = applicationMasterConfig.getMainClass(serviceName);
    String zkAddress = applicationMasterConfig.getZKAddress();

    // set the localresources needed to launch container
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();

    LocalResource servicePackageResource = Records.newRecord(LocalResource.class);
    YarnConfiguration conf = new YarnConfiguration();
    FileSystem fs;/*from   ww  w.ja  v  a2s  .  co m*/
    fs = FileSystem.get(conf);
    String pathSuffix = appName + "/" + appId + "/" + serviceName + ".tar";
    Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
    FileStatus destStatus = fs.getFileStatus(dst);

    // Set the type of resource - file or archive
    // archives are untarred at destination
    // we don't need the jar file to be untarred for now
    servicePackageResource.setType(LocalResourceType.ARCHIVE);
    // Set visibility of the resource
    // Setting to most private option
    servicePackageResource.setVisibility(LocalResourceVisibility.APPLICATION);
    // Set the resource to be copied over
    servicePackageResource.setResource(ConverterUtils.getYarnUrlFromPath(dst));
    // Set timestamp and length of file so that the framework
    // can do basic sanity checks for the local resource
    // after it has been copied over to ensure it is the same
    // resource the client intended to use with the application
    servicePackageResource.setTimestamp(destStatus.getModificationTime());
    servicePackageResource.setSize(destStatus.getLen());
    LOG.info("Setting local resource:" + servicePackageResource + " for service" + serviceName);
    localResources.put(serviceName, servicePackageResource);

    // Set local resource info into app master container launch context
    participantContainer.setLocalResources(localResources);

    // Set the necessary security tokens as needed
    // amContainer.setContainerTokens(containerToken);

    // Set the env variables to be setup in the env where the application master will be run
    LOG.info("Set the environment for the application master");
    Map<String, String> env = new HashMap<String, String>();
    env.put(serviceName, dst.getName());
    // Add AppMaster.jar location to classpath
    // At some point we should not be required to add
    // the hadoop specific classpaths to the env.
    // It should be provided out of the box.
    // For now setting all required classpaths including
    // the classpath to "." for the application jar
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar)
            .append("./*");
    classPathEnv.append(File.pathSeparatorChar);
    classPathEnv.append(serviceClasspath);
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
    LOG.info("Setting classpath for service:\n" + classPathEnv.toString());
    env.put("CLASSPATH", classPathEnv.toString());

    participantContainer.setEnvironment(env);

    if (applicationMaster.allTokens != null) {
        LOG.info("Setting tokens: " + applicationMaster.allTokens);
        participantContainer.setTokens(applicationMaster.allTokens);
    }

    // Set the necessary command to execute the application master
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);

    // Set java executable command
    LOG.info("Setting up app master command");
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx" + 4096 + "m");
    // Set class name
    vargs.add(ParticipantLauncher.class.getCanonicalName());
    // Set params for container participant
    vargs.add("--zkAddress " + zkAddress);
    vargs.add("--cluster " + appName);
    vargs.add("--participantId " + participant.getId().stringify());
    vargs.add("--participantClass " + mainClass);

    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/ContainerParticipant.stderr");

    // Get final commmand
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up  container launch command " + command.toString() + " with arguments \n"
            + vargs);
    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());
    participantContainer.setCommands(commands);
    return participantContainer;
}

From source file:org.apache.ignite.yarn.IgniteYarnClient.java

License:Apache License

/**
 * @param envs Environment variables./*  w w  w.j  a v a2  s.c om*/
 * @param conf Yarn configuration.
 */
private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
        Apps.addToEnvironment(envs, Environment.CLASSPATH.name(), c.trim(), File.pathSeparator);

    Apps.addToEnvironment(envs, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*",
            File.pathSeparator);
}

From source file:org.apache.sysml.yarn.DMLYarnClient.java

License:Apache License

private static Map<String, String> constructEnvionmentMap(YarnConfiguration yconf) throws IOException {
    Map<String, String> eMap = new HashMap<>();

    //setup default app master environment
    StringBuilder classpath = new StringBuilder();
    for (String value : yconf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(value.trim());/* w w w .  j  a  v a  2 s  .c  o m*/
    }

    //setup mapreduce appmaster environment (for robustness if not included in default environment)
    //for example, by default HDP 2.2 did not include mapred client libraries in this configuration
    //note: we cannot use MRConfigurationNames.MR_APPLICATION_CLASSPATH because it refers to HDFS and $PWD that needs to be setup
    Map<String, String> env = System.getenv();
    String mapred_home = null;
    //get mapred home via alternative environment variables
    if (env.containsKey(MAPRED_HOME_ENV_CONST)) {
        mapred_home = env.get(MAPRED_HOME_ENV_CONST);
    } else if (env.containsKey(HADOOP_HOME_ENV_CONST)) {
        String tmp = env.get(HADOOP_HOME_ENV_CONST);
        mapred_home = tmp + File.separator + ".." + File.separator + "hadoop-mapreduce";
    }
    //concatenate mapred home libs to classpath
    if (mapred_home != null) {
        if (classpath.length() > 0)
            classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "*");
        classpath.append(File.pathSeparator);
        classpath.append(mapred_home + File.separator + "lib" + File.separator + "*");
    }

    eMap.put(Environment.CLASSPATH.name(), classpath.toString());
    MRApps.setClasspath(eMap, yconf);

    LOG.debug("Constructed environment classpath: " + classpath.toString());

    return eMap;
}