Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME.

Prototype

String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME.

Click Source Link

Document

If the port should be included or not in the node name.

Usage

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

public static Configuration createMiniLlamaConf(Configuration conf, int nodes) {
    ParamChecker.notNull(conf, "conf");
    ParamChecker.greaterThan(nodes, 0, "nodes");
    conf.set(ServerConfiguration.CONFIG_DIR_KEY, "");
    conf.setIfUnset(LlamaAM.RM_CONNECTOR_CLASS_KEY, YarnRMConnector.class.getName());
    conf.setInt(MINI_CLUSTER_NODES_KEY, nodes);
    conf.setIfUnset(S_CONF.getPropertyName(ServerConfiguration.SERVER_ADDRESS_KEY), "localhost:0");
    conf.setIfUnset(S_CONF.getPropertyName(ServerConfiguration.SERVER_ADMIN_ADDRESS_KEY), "localhost:0");
    conf.setIfUnset(S_CONF.getPropertyName(ServerConfiguration.HTTP_ADDRESS_KEY), "localhost:0");
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);

    conf.set(YarnRMConnector.HADOOP_USER_NAME_KEY, System.getProperty("user.name"));
    return conf;/* w w w .j  a v a  2s .  com*/
}

---

From source file:com.cloudera.llama.am.MiniLlama.java

License:Apache License

private Configuration startMiniHadoop() throws Exception {
    int clusterNodes = getConf().getInt(MINI_CLUSTER_NODES_KEY, 1);
    if (System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA) == null) {
        String testBuildData = new File("target").getAbsolutePath();
        System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testBuildData);
    }/* w w  w  .j a va 2  s  . c o m*/
    //to trigger hdfs-site.xml registration as default resource
    new HdfsConfiguration();
    Configuration conf = new YarnConfiguration();
    String llamaProxyUser = System.getProperty("user.name");
    conf.set("hadoop.security.authentication", "simple");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*");
    String[] userGroups = new String[] { "g" };
    UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups);

    int hdfsPort = 0;
    String fsUri = conf.get("fs.defaultFS");
    if (fsUri != null && !fsUri.equals("file:///")) {
        int i = fsUri.lastIndexOf(":");
        if (i > -1) {
            try {
                hdfsPort = Integer.parseInt(fsUri.substring(i + 1));
            } catch (Exception ex) {
                throw new RuntimeException(
                        "Could not parse port from Hadoop's " + "'fs.defaultFS property: " + fsUri);
            }
        }
    }
    miniHdfs = new MiniDFSCluster(hdfsPort, conf, clusterNodes, !skipDfsFormat, true, null, null);
    miniHdfs.waitActive();
    conf = miniHdfs.getConfiguration(0);
    miniYarn = new MiniYARNCluster("minillama", clusterNodes, 1, 1);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);

    miniYarn.init(conf);
    miniYarn.start();
    conf = miniYarn.getConfig();

    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
    return conf;
}

---

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

private Configuration createMiniYarnConfig(boolean usePortInName) throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set("yarn.scheduler.fair.allocation.file", "test-fair-scheduler.xml");
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, FairScheduler.class);

    //proxy user config
    String llamaProxyUser = System.getProperty("user.name");
    conf.set("hadoop.security.authentication", "simple");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*");
    conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*");
    String[] userGroups = new String[] { "g" };
    UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups);
    conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, usePortInName);
    return conf;/*from   w  w w .  j a v  a  2s .  c o  m*/
}

---

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

@Override
public void setConf(Configuration conf) {
    this.conf = conf;
    includePortInNodeName = getConf().getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
    appType = getConf().get(LlamaAM.CLUSTER_ID, LlamaAM.CLUSTER_ID_DEFAULT);
    yarnConf = new YarnConfiguration();
    for (Map.Entry entry : getConf()) {
        yarnConf.set((String) entry.getKey(), (String) entry.getValue());
    }//  www .  j a  v  a 2  s. c o m
}

---