Example usage for org.apache.hadoop.yarn.server MiniYARNCluster MiniYARNCluster

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.server MiniYARNCluster MiniYARNCluster.

Prototype

public MiniYARNCluster(String testName, int numResourceManagers, int numNodeManagers, int numLocalDirs,
        int numLogDirs) 

Source Link

Usage

From source file:co.cask.cdap.operations.yarn.YarnRMHAOperationalStatsTest.java

License:Apache License

@Override
protected MiniYARNCluster createYarnCluster() throws IOException, InterruptedException, YarnException {
    Configuration hConf = new Configuration();
    hConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    String hostname = MiniYARNCluster.getHostname();
    for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
        hConf.set(HAUtil.addSuffix(confKey, "rm0"), hostname + ":" + Networks.getRandomPort());
        hConf.set(HAUtil.addSuffix(confKey, "rm1"), hostname + ":" + Networks.getRandomPort());
    }/*from ww w .  j av a  2 s . c  om*/
    MiniYARNCluster yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 2, 2, 2);
    yarnCluster.init(hConf);
    yarnCluster.start();
    yarnCluster.getResourceManager(0).getRMContext().getRMAdminService().transitionToActive(
            new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER));
    return yarnCluster;
}

---

From source file:io.hops.tensorflow.TestCluster.java

License:Apache License

protected void setupInternal(int numNodeManager) throws Exception {

    LOG.info("Starting up YARN cluster");

    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.set("yarn.log.dir", "target");
    conf.set("yarn.log-aggregation-enable", "true");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false);

    if (yarnCluster == null) {
        yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1);
        yarnCluster.init(conf);/* ww w . j av a2s .  c o m*/

        yarnCluster.start();

        conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
                MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort());

        waitForNMsToRegister();

        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
        //write the document to a buffer (not directly to the file, as that
        //can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        //write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
    }
    FileContext fsContext = FileContext.getLocalFSFileContext();
    fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
        LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
    }
}

---