Example usage for org.apache.hadoop.yarn.server MiniYARNCluster MiniYARNCluster

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.server MiniYARNCluster MiniYARNCluster.

Prototype

public MiniYARNCluster(String testName, int numNodeManagers, int numLocalDirs, int numLogDirs) 

Source Link

Usage

From source file:org.apache.zeppelin.integration.MiniHadoopCluster.java

License:Apache License

@BeforeClass
public void start() throws IOException {
    LOGGER.info("Starting MiniHadoopCluster ...");
    this.hadoopConf = new Configuration();
    new File(configPath).mkdirs();
    // start MiniDFSCluster
    this.dfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(2).format(true).waitSafeMode(true)
            .build();/*from   w  w  w . ja v  a2 s  . c o  m*/
    this.dfsCluster.waitActive();
    saveConfig(hadoopConf, configPath + "/core-site.xml");

    // start MiniYarnCluster
    YarnConfiguration baseConfig = new YarnConfiguration(hadoopConf);
    baseConfig.set("yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage", "95");
    this.yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 1, 1);
    yarnCluster.init(baseConfig);

    // Install a shutdown hook for stop the service and kill all running applications.
    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            yarnCluster.stop();
        }
    });

    yarnCluster.start();

    // Workaround for YARN-2642.
    Configuration yarnConfig = yarnCluster.getConfig();
    long start = System.currentTimeMillis();
    while (System.currentTimeMillis() - start < 30 * 1000) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
        if (!yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")[1].equals("0")) {
            break;
        }
    }
    if (yarnConfig.get(YarnConfiguration.RM_ADDRESS).split(":")[1].equals("0")) {
        throw new IOException("RM not up yes");
    }

    LOGGER.info("RM address in configuration is " + yarnConfig.get(YarnConfiguration.RM_ADDRESS));
    saveConfig(yarnConfig, configPath + "/yarn-site.xml");
}

---

From source file:org.elasticsearch.hadoop.integration.yarn.YarnTestCluster.java

License:Apache License

@Override
protected void before() throws Throwable {
    if (yarnCluster != null) {
        return;//from   w w  w  .ja  v  a  2s .  c  o  m
    }

    System.out.println("Starting YARN cluster...");

    //tempDir = System.getProperty("java.io.tmpdir");
    //System.setProperty("java.io.tmpdir", "/tmp-yarn");

    //        FileContext.DEFAULT_PERM.fromShort((short) 777);
    //        FileContext.DIR_DEFAULT_PERM.fromShort((short) 777);
    //        FileContext.FILE_DEFAULT_PERM.fromShort((short) 777);
    //ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION.fromShort((short) 777);

    // make sure to use IP discovery
    Configuration.addDefaultResource("yarn-test-site.xml");

    cfg.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "build/es-yarn/" + CLUSTER_NAME + "-dfs");

    dfsCluster = new MiniDFSCluster.Builder(cfg).numDataNodes(nodes).build();
    System.out.println("Started DFS cluster...");

    cfg.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    yarnCluster = new MiniYARNCluster(CLUSTER_NAME, nodes, 1, 1);
    yarnCluster.init(cfg);
    yarnCluster.start();
    System.out.println("Started YARN cluster...");
}

---

From source file:org.springframework.yarn.test.support.StandaloneYarnCluster.java

License:Apache License

@Override
public void start() throws IOException {
    log.info("Checking if cluster=" + clusterName + " needs to be started");
    synchronized (this.startupShutdownMonitor) {
        if (started) {
            return;
        }/*from  w  w  w . j ava  2s  . com*/
        log.info("Starting cluster=" + clusterName);
        configuration = new YarnConfiguration();
        configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs");

        dfsCluster = new MiniDFSCluster.Builder(configuration).numDataNodes(nodes).build();

        yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1);
        yarnCluster.init(configuration);
        yarnCluster.start();

        log.info("Started cluster=" + clusterName);
        started = true;
    }
}

---

From source file:probos.TestEndToEnd.java

License:Open Source License

@Before
public void setupCluster() throws Exception {
    System.setProperty("probos.home", System.getProperty("user.dir"));
    (probosJobDir = new File(System.getProperty("user.dir"), "probos")).mkdir();
    String name = "mycluster";
    int noOfNodeManagers = 1;
    int numLocalDirs = 1;
    int numLogDirs = 1;
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs);
    miniCluster.init(conf);//from www . j  av  a  2  s. c o  m
    miniCluster.start();

    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    YarnConfiguration appConf = new YarnConfiguration(miniCluster.getConfig());
    cs = new ControllerServer(appConf);
    cs.startAndWait();
    Thread.sleep(1000);
    new pbsnodes().run(new String[0]);
}

---

From source file:probos.TestProblem.java

License:Open Source License

@Before
public void setupCluster() throws Exception {
    String name = "mycluster";
    int noOfNodeManagers = 1;
    int numLocalDirs = 1;
    int numLogDirs = 1;
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs);
    miniCluster.init(conf);/*from   w  w w  .java 2  s. c  o m*/
    miniCluster.start();

    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    yConf = new YarnConfiguration(miniCluster.getConfig());

}

---

• «
• 1
• 2
• 3
• »