Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER.

Prototype

String RM_SCHEDULER

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER.

Click Source Link

Document

The class to use as the resource scheduler.

Usage

From source file:org.zuinnote.hadoop.office.example.MapReduceExcelInputIntegrationTest.java

License:Apache License

@BeforeAll
public static void oneTimeSetUp() throws IOException {
    // Create temporary directory for HDFS base and shutdownhook 
    // create temp directory
    tmpPath = Files.createTempDirectory(tmpPrefix);
    // create shutdown hook to remove temp files (=HDFS MiniCluster) after shutdown, may need to rethink to avoid many threads are created
    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        @Override/* w  w w  .j  a v a2  s . co m*/
        public void run() {
            try {
                Files.walkFileTree(tmpPath, new SimpleFileVisitor<java.nio.file.Path>() {

                    @Override
                    public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs)
                            throws IOException {
                        Files.delete(file);
                        return FileVisitResult.CONTINUE;
                    }

                    @Override
                    public FileVisitResult postVisitDirectory(java.nio.file.Path dir, IOException e)
                            throws IOException {
                        if (e == null) {
                            Files.delete(dir);
                            return FileVisitResult.CONTINUE;
                        }
                        throw e;
                    }
                });
            } catch (IOException e) {
                throw new RuntimeException(
                        "Error temporary files in following path could not be deleted " + tmpPath, e);
            }
        }
    }));
    // Create Configuration
    Configuration conf = new Configuration();
    // create HDFS cluster
    File baseDir = new File(tmpPath.toString()).getAbsoluteFile();
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
    dfsCluster = builder.numDataNodes(NOOFDATANODES).build();
    // create Yarn cluster
    YarnConfiguration clusterConf = new YarnConfiguration(conf);
    conf.set("fs.defaultFS", dfsCluster.getFileSystem().getUri().toString());
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    miniCluster = new MiniMRYarnCluster(CLUSTERNAME, NOOFNODEMANAGERS, STARTTIMELINESERVER);
    miniCluster.init(conf);
    miniCluster.start();
}

---

From source file:oz.hadoop.yarn.api.core.ApplicationMasterLauncherImpl.java

License:Apache License

/**
 * Any type of pre-check you want to perform before launching Application Master
 * mainly for the purpose of logging warning messages
 *///from   w w  w.  j a  v  a  2 s.co m
private void preCheck() {
    if (this.applicationContainerSpecification.getInt(YayaConstants.VIRTUAL_CORES) > 1) {
        if (!this.yarnConfig.get(YarnConfiguration.RM_SCHEDULER).equals(FairScheduler.class.getName())) {
            logger.warn("Based on current Hadoop implementation "
                    + "'vcore' settings are ignored for schedulers other then FairScheduler");
        }
    }
    try {
        Iterator<QueueInfo> queues = this.yarnClient.getAllQueues().iterator();
        String identifiedQueueName = (String) this.applicationSpecification.get(YayaConstants.QUEUE_NAME);
        boolean queueExist = false;
        while (!queueExist && queues.hasNext()) {
            QueueInfo queueInfo = queues.next();
            if (queueInfo.getQueueName().equals(identifiedQueueName)) {
                queueExist = true;
            }
        }
        if (!queueExist) {
            throw new IllegalArgumentException("Queue with the name '" + identifiedQueueName
                    + "' does not exist. Aborting application launch.");
        }
    } catch (Exception e) {
        throw new IllegalStateException("Failed to validate queue.", e);
    }
}

---

From source file:probos.TestEndToEnd.java

License:Open Source License

@Before
public void setupCluster() throws Exception {
    System.setProperty("probos.home", System.getProperty("user.dir"));
    (probosJobDir = new File(System.getProperty("user.dir"), "probos")).mkdir();
    String name = "mycluster";
    int noOfNodeManagers = 1;
    int numLocalDirs = 1;
    int numLogDirs = 1;
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs);
    miniCluster.init(conf);/*w  w  w  . j av a  2 s.  c om*/
    miniCluster.start();

    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    YarnConfiguration appConf = new YarnConfiguration(miniCluster.getConfig());
    cs = new ControllerServer(appConf);
    cs.startAndWait();
    Thread.sleep(1000);
    new pbsnodes().run(new String[0]);
}

---

From source file:probos.TestProblem.java

License:Open Source License

@Before
public void setupCluster() throws Exception {
    String name = "mycluster";
    int noOfNodeManagers = 1;
    int numLocalDirs = 1;
    int numLogDirs = 1;
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs);
    miniCluster.init(conf);//from  w w w. jav a2  s  .  c om
    miniCluster.start();

    //once the cluster is created, you can get its configuration
    //with the binding details to the cluster added from the minicluster
    yConf = new YarnConfiguration(miniCluster.getConfig());

}

---

• «
• 1
• 2
• 3
• »