Example usage for org.apache.hadoop.mapreduce.v2.jobhistory JHAdminConfig MR_HISTORY_MINICLUSTER_FIXED

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce.v2.jobhistory JHAdminConfig MR_HISTORY_MINICLUSTER_FIXED_PORTS.

Prototype

String MR_HISTORY_MINICLUSTER_FIXED_PORTS

To view the source code for org.apache.hadoop.mapreduce.v2.jobhistory JHAdminConfig MR_HISTORY_MINICLUSTER_FIXED_PORTS.

Click Source Link

Document

Whether to use fixed ports with the minicluster.

Usage

From source file:com.github.sakserv.minicluster.impl.MRLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {

    // Handle Windows
    WindowsLibsUtils.setHadoopHome();//from   ww w . jav a2s .  c  o  m

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(JHAdminConfig.MR_HISTORY_ADDRESS, jobHistoryAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    configuration.set(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }

    if (null != hdfsDefaultFs) {
        configuration.set("fs.defaultFS", hdfsDefaultFs);
        configuration.set("dfs.replication", "1");
    }
}

From source file:com.google.mr4c.hadoop.yarn.YarnTestBinding.java

License:Open Source License

private void startMrCluster() throws IOException {
    Configuration conf = new JobConf();
    FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri());
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    String addr = MiniYARNCluster.getHostname() + ":0";
    conf.set(YarnConfiguration.RM_ADDRESS, addr);
    conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr);
    m_mrCluster = MiniMRClientClusterFactory.create(HadoopTestUtils.class, "MR4CTests", 1, // num node managers
            conf);/*from  w w  w.j av  a2  s. c  o  m*/

    // make sure startup is finished
    for (int i = 0; i < 60; i++) {
        String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS);
        if (newAddr.equals(addr)) {
            s_log.warn("MiniYARNCluster startup not complete");
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
                throw new IOException(ie);
            }
        } else {
            s_log.info("MiniYARNCluster now available at {}", newAddr);
            return;
        }
    }
    throw new IOException("MiniYARNCluster taking too long to startup");

}

From source file:com.ikanow.aleph2.analytics.hadoop.services.MiniClusterBeJobLauncher.java

License:Apache License

@Override
public Configuration getHadoopConfig() {
    if (_configuration == null) {

        this._configuration = new Configuration(true);
        String stagingdir = _configuration.get("yarn.app.mapreduce.am.staging-dir");
        logger.debug("staging dir:" + stagingdir);
        _configuration.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
        _configuration.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);

        try {// w ww.  j  av a 2 s. co  m
            mrCluster = new MiniMRYarnCluster("MiniClusterTest", 1);
            mrCluster.init(_configuration);
            start();
        } catch (Exception e) {
            logger.error("getConfiguration caused exception", e);
        }
    }
    return _configuration;
}

From source file:org.apache.solr.hadoop.hack.MiniMRYarnClusterAdapter.java

License:Apache License

@Override
public void restart() {
    if (!miniMRYarnCluster.getServiceState().equals(STATE.STARTED)) {
        LOG.warn("Cannot restart the mini cluster, start it first");
        return;/*  www  . j  a v a  2 s.  com*/
    }
    Configuration oldConf = new Configuration(getConfig());
    String callerName = oldConf.get("minimrclientcluster.caller.name", this.getClass().getName());
    int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1);
    oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    stop();
    miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs, testWorkDir);
    miniMRYarnCluster.init(oldConf);
    miniMRYarnCluster.start();
}

Example usage for org.apache.hadoop.mapreduce.v2.jobhistory JHAdminConfig MR_HISTORY_MINICLUSTER_FIXED_PORTS

Introduction

Prototype

Document

Usage