Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_MINICLUSTER_FIXED

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_MINICLUSTER_FIXED_PORTS.

Prototype

String YARN_MINICLUSTER_FIXED_PORTS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration YARN_MINICLUSTER_FIXED_PORTS.

Click Source Link

Document

Whether to use fixed ports with the minicluster.

Usage

From source file:co.cask.cdap.operations.yarn.YarnRMHAOperationalStatsTest.java

License:Apache License

@Override
protected MiniYARNCluster createYarnCluster() throws IOException, InterruptedException, YarnException {
    Configuration hConf = new Configuration();
    hConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    String hostname = MiniYARNCluster.getHostname();
    for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
        hConf.set(HAUtil.addSuffix(confKey, "rm0"), hostname + ":" + Networks.getRandomPort());
        hConf.set(HAUtil.addSuffix(confKey, "rm1"), hostname + ":" + Networks.getRandomPort());
    }/*from www . java  2 s .  co m*/
    MiniYARNCluster yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 2, 2, 2);
    yarnCluster.init(hConf);
    yarnCluster.start();
    yarnCluster.getResourceManager(0).getRMContext().getRMAdminService().transitionToActive(
            new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER));
    return yarnCluster;
}

From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java

License:Apache License

private void restartMiniYarn() throws Exception {
    Configuration conf = miniYarn.getConfig();
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    stopYarn();/*w w  w. j a va2 s.  c  o m*/
    startYarn(conf);
}

From source file:com.github.sakserv.minicluster.impl.MRLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {

    // Handle Windows
    WindowsLibsUtils.setHadoopHome();//from  w w w .j  a v  a2 s  .c o  m

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(JHAdminConfig.MR_HISTORY_ADDRESS, jobHistoryAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    configuration.set(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }

    if (null != hdfsDefaultFs) {
        configuration.set("fs.defaultFS", hdfsDefaultFs);
        configuration.set("dfs.replication", "1");
    }
}

From source file:com.github.sakserv.minicluster.impl.YarnLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {
    // Handle Windows
    WindowsLibsUtils.setHadoopHome();//from   www. ja v a2 s .c  o m

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }
}

From source file:com.google.mr4c.hadoop.yarn.YarnTestBinding.java

License:Open Source License

private void startMrCluster() throws IOException {
    Configuration conf = new JobConf();
    FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri());
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    String addr = MiniYARNCluster.getHostname() + ":0";
    conf.set(YarnConfiguration.RM_ADDRESS, addr);
    conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr);
    m_mrCluster = MiniMRClientClusterFactory.create(HadoopTestUtils.class, "MR4CTests", 1, // num node managers
            conf);/*from  w w  w .  ja  v a  2  s.  c  o m*/

    // make sure startup is finished
    for (int i = 0; i < 60; i++) {
        String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS);
        if (newAddr.equals(addr)) {
            s_log.warn("MiniYARNCluster startup not complete");
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
                throw new IOException(ie);
            }
        } else {
            s_log.info("MiniYARNCluster now available at {}", newAddr);
            return;
        }
    }
    throw new IOException("MiniYARNCluster taking too long to startup");

}

From source file:com.ikanow.aleph2.analytics.hadoop.services.MiniClusterBeJobLauncher.java

License:Apache License

@Override
public Configuration getHadoopConfig() {
    if (_configuration == null) {

        this._configuration = new Configuration(true);
        String stagingdir = _configuration.get("yarn.app.mapreduce.am.staging-dir");
        logger.debug("staging dir:" + stagingdir);
        _configuration.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
        _configuration.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);

        try {/*from  ww  w .jav a  2s  .co  m*/
            mrCluster = new MiniMRYarnCluster("MiniClusterTest", 1);
            mrCluster.init(_configuration);
            start();
        } catch (Exception e) {
            logger.error("getConfiguration caused exception", e);
        }
    }
    return _configuration;
}

From source file:org.apache.giraph.yarn.TestYarnJob.java

License:Apache License

/**
 * Set up the GiraphConfiguration settings we need to run a no-op Giraph
 * job on a MiniYARNCluster as an integration test. Some YARN-specific
 * flags are set inside GiraphYarnClient and won't need to be set here.
 *//*from   w  w  w  . j ava2s  .co  m*/
private void setupYarnConfiguration() throws IOException {
    conf = new GiraphConfiguration();
    conf.setWorkerConfiguration(1, 1, 100.0f);
    conf.setMaxMasterSuperstepWaitMsecs(30 * 1000);
    conf.setEventWaitMsecs(3 * 1000);
    conf.setYarnLibJars(""); // no need
    conf.setYarnTaskHeapMb(256); // small since no work to be done
    conf.setComputationClass(DummyYarnComputation.class);
    conf.setVertexInputFormatClass(IntIntNullTextInputFormat.class);
    conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class);
    conf.setNumComputeThreads(1);
    conf.setMaxTaskAttempts(1);
    conf.setNumInputSplitsThreads(1);
    // Giraph on YARN only ever things its running in "non-local" mode
    conf.setLocalTestMode(false);
    // this has to happen here before we populate the conf with the temp dirs
    setupTempDirectories();
    conf.set(OUTDIR, new Path(outputDir.getAbsolutePath()).toString());
    GiraphFileInputFormat.addVertexInputPath(conf, new Path(inputDir.getAbsolutePath()));
    // hand off the ZK info we just created to our no-op job
    GiraphConstants.ZOOKEEPER_SERVERLIST_POLL_MSECS.set(conf, 500);
    conf.setZooKeeperConfiguration(zkList);
    conf.set(GiraphConstants.ZOOKEEPER_DIR, zkDir.getAbsolutePath());
    GiraphConstants.ZOOKEEPER_MANAGER_DIRECTORY.set(conf, zkMgrDir.getAbsolutePath());
    // without this, our "real" client won't connect w/"fake" YARN cluster
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
}

From source file:org.apache.metron.integration.components.MRComponent.java

License:Apache License

@Override
public void start() {
    configuration = new Configuration();
    System.clearProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    if (basePath == null) {
        throw new RuntimeException("Unable to start cluster: You must specify the basepath");
    }/* www  .j  av a2 s . c  om*/
    configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, basePath.toString());
    try {
        cluster = new MiniDFSCluster.Builder(configuration).build();
    } catch (IOException e) {
        throw new RuntimeException("Unable to start cluster", e);
    }
}

From source file:org.apache.solr.hadoop.hack.MiniMRYarnClusterAdapter.java

License:Apache License

@Override
public void restart() {
    if (!miniMRYarnCluster.getServiceState().equals(STATE.STARTED)) {
        LOG.warn("Cannot restart the mini cluster, start it first");
        return;/*from   w w w.j  av a  2  s  .  c o  m*/
    }
    Configuration oldConf = new Configuration(getConfig());
    String callerName = oldConf.get("minimrclientcluster.caller.name", this.getClass().getName());
    int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1);
    oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    stop();
    miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs, testWorkDir);
    miniMRYarnCluster.init(oldConf);
    miniMRYarnCluster.start();
}

From source file:org.elasticsearch.hadoop.integration.yarn.YarnTestCluster.java

License:Apache License

@Override
protected void before() throws Throwable {
    if (yarnCluster != null) {
        return;/*from  ww w  . ja  v  a2 s .  c o  m*/
    }

    System.out.println("Starting YARN cluster...");

    //tempDir = System.getProperty("java.io.tmpdir");
    //System.setProperty("java.io.tmpdir", "/tmp-yarn");

    //        FileContext.DEFAULT_PERM.fromShort((short) 777);
    //        FileContext.DIR_DEFAULT_PERM.fromShort((short) 777);
    //        FileContext.FILE_DEFAULT_PERM.fromShort((short) 777);
    //ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION.fromShort((short) 777);

    // make sure to use IP discovery
    Configuration.addDefaultResource("yarn-test-site.xml");

    cfg.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "build/es-yarn/" + CLUSTER_NAME + "-dfs");

    dfsCluster = new MiniDFSCluster.Builder(cfg).numDataNodes(nodes).build();
    System.out.println("Started DFS cluster...");

    cfg.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    yarnCluster = new MiniYARNCluster(CLUSTER_NAME, nodes, 1, 1);
    yarnCluster.init(cfg);
    yarnCluster.start();
    System.out.println("Started YARN cluster...");
}

Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YARN_MINICLUSTER_FIXED_PORTS

Introduction

Prototype

Document

Usage