Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RESOURCEMANAGER_CONNECT_MAX_WAIT_MS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RESOURCEMANAGER_CONNECT_MAX_WAIT_MS.

Prototype

String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RESOURCEMANAGER_CONNECT_MAX_WAIT_MS.

Click Source Link

Document

Max time to wait to establish a connection to RM

Usage

From source file:com.datatorrent.stram.client.StramClientUtils.java

License:Apache License

public static Configuration addDTSiteResources(Configuration conf) {
    addDTLocalResources(conf);/*from w  ww  .  ja  v a 2s . co  m*/
    FileSystem fs = null;
    File targetGlobalFile;
    try {
        fs = newFileSystemInstance(conf);
        // after getting the dfsRootDirectory config parameter, redo the entire process with the global config
        // load global settings from DFS
        targetGlobalFile = new File(String.format("/tmp/dt-site-global-%s.xml",
                UserGroupInformation.getLoginUser().getShortUserName()));
        org.apache.hadoop.fs.Path hdfsGlobalPath = new org.apache.hadoop.fs.Path(
                StramClientUtils.getDTDFSConfigDir(fs, conf), StramClientUtils.DT_SITE_GLOBAL_XML_FILE);
        LOG.debug("Copying global dt-site.xml from {} to {}", hdfsGlobalPath,
                targetGlobalFile.getAbsolutePath());
        fs.copyToLocalFile(hdfsGlobalPath, new org.apache.hadoop.fs.Path(targetGlobalFile.toURI()));
        addDTSiteResources(conf, targetGlobalFile);
        if (!isDevelopmentMode()) {
            // load node local config file
            addDTSiteResources(conf,
                    new File(StramClientUtils.getConfigDir(), StramClientUtils.DT_SITE_XML_FILE));
        }
        // load user config file
        addDTSiteResources(conf,
                new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE));
    } catch (IOException ex) {
        // ignore
        LOG.debug("Caught exception when loading configuration: {}: moving on...", ex.getMessage());
    } finally {
        // Cannot delete the file here because addDTSiteResource which eventually calls Configuration.reloadConfiguration
        // does not actually reload the configuration.  The file is actually read later and it needs to exist.
        //
        //if (targetGlobalFile != null) {
        //targetGlobalFile.delete();
        //}
        IOUtils.closeQuietly(fs);
    }

    //Validate loggers-level settings
    String loggersLevel = conf.get(DTLoggerFactory.DT_LOGGERS_LEVEL);
    if (loggersLevel != null) {
        String targets[] = loggersLevel.split(",");
        Preconditions.checkArgument(targets.length > 0, "zero loggers level");
        for (String target : targets) {
            String parts[] = target.split(":");
            Preconditions.checkArgument(parts.length == 2, "incorrect " + target);
            Preconditions.checkArgument(ConfigValidator.validateLoggersLevel(parts[0], parts[1]),
                    "incorrect " + target);
        }
    }
    convertDeprecatedProperties(conf);

    //
    // The ridiculous default RESOURCEMANAGER_CONNECT_MAX_WAIT_MS from hadoop is 15 minutes (!!!!), which actually translates to 20 minutes with the connect interval.
    // That means if there is anything wrong with YARN or if YARN is not running, the caller has to wait for up to 20 minutes until it gets an error.
    // We are overriding this to be 10 seconds maximum.
    //

    int rmConnectMaxWait = conf.getInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
            YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
    if (rmConnectMaxWait > RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE) {
        LOG.info("Overriding {} assigned value of {} to {} because the assigned value is too big.",
                YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, rmConnectMaxWait,
                RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE);
        conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
                RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE);
        int rmConnectRetryInterval = conf.getInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
                YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
        int defaultRetryInterval = Math.max(500, RESOURCEMANAGER_CONNECT_MAX_WAIT_MS_OVERRIDE / 5);
        if (rmConnectRetryInterval > defaultRetryInterval) {
            LOG.info("Overriding {} assigned value of {} to {} because the assigned value is too big.",
                    YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, rmConnectRetryInterval,
                    defaultRetryInterval);
            conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, defaultRetryInterval);
        }
    }
    LOG.info(" conf object in stramclient {}", conf);
    return conf;
}

---

From source file:org.apache.gobblin.yarn.YarnServiceTest.java

License:Apache License

@BeforeClass
public void setUp() throws Exception {
    // Set java home in environment since it isn't set on some systems
    String javaHome = System.getProperty("java.home");
    setEnv("JAVA_HOME", javaHome);

    this.clusterConf = new YarnConfiguration();
    this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100");
    this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000");
    this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000");

    this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1));
    this.yarnCluster.init(this.clusterConf);
    this.yarnCluster.start();

    // YARN client should not be started before the Resource Manager is up
    AssertWithBackoff.create().logger(LOG).timeoutMs(10000).assertTrue(new Predicate<Void>() {
        @Override/*from w w w  .  j  a v a  2  s  .c o  m*/
        public boolean apply(Void input) {
            return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
    }, "Waiting for RM");

    this.yarnClient = this.closer.register(YarnClient.createYarnClient());
    this.yarnClient.init(this.clusterConf);
    this.yarnClient.start();

    URL url = YarnServiceTest.class.getClassLoader()
            .getResource(YarnServiceTest.class.getSimpleName() + ".conf");
    Assert.assertNotNull(url, "Could not find resource " + url);

    this.config = ConfigFactory.parseURL(url).resolve();

    // Start a dummy application manager so that the YarnService can use the AM-RM token.
    startApp();

    // create and start the test yarn service
    this.yarnService = new TestYarnService(this.config, "testApp", "appId", this.clusterConf,
            FileSystem.getLocal(new Configuration()), this.eventBus);

    this.yarnService.startUp();
}

---