Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER.

Prototype

String RM_SCHEDULER

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER.

Click Source Link

Document

The class to use as the resource scheduler.

Usage

From source file:io.hops.tensorflow.TestCluster.java

License:Apache License

protected void setupInternal(int numNodeManager) throws Exception {

    LOG.info("Starting up YARN cluster");

    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.set("yarn.log.dir", "target");
    conf.set("yarn.log-aggregation-enable", "true");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false);

    if (yarnCluster == null) {
        yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1);
        yarnCluster.init(conf);//from  w ww.j  av a2  s  .  co  m

        yarnCluster.start();

        conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
                MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort());

        waitForNMsToRegister();

        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
        //write the document to a buffer (not directly to the file, as that
        //can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        //write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
    }
    FileContext fsContext = FileContext.getLocalFSFileContext();
    fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
        LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
    }
}

From source file:org.apache.asterix.aoya.test.YARNCluster.java

License:Apache License

/**
 * Instantiates the (Mini) DFS Cluster with the configured number of datanodes.
 * Post instantiation, data is laoded to HDFS.
 * Called prior to running the Runtime test suite.
 *///www .  ja v  a 2s  .  c o m
public void setup() throws Exception {
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
    conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/integrationts/data");
    cleanupLocal();
    //this constructor is deprecated in hadoop 2x
    //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null);
    miniCluster = new MiniYARNCluster("Asterix_testing", numDataNodes, 1, 1);
    miniCluster.init(conf);
}

From source file:org.apache.flink.yarn.YARNSessionCapacitySchedulerITCase.java

License:Apache License

@BeforeClass
public static void setup() {
    yarnConfiguration.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
            ResourceScheduler.class);
    yarnConfiguration.set("yarn.scheduler.capacity.root.queues", "default,qa-team");
    yarnConfiguration.setInt("yarn.scheduler.capacity.root.default.capacity", 40);
    yarnConfiguration.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60);
    yarnConfiguration.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler");
    startYARNWithConfig(yarnConfiguration);
}

From source file:org.apache.flink.yarn.YARNSessionFIFOITCase.java

License:Apache License

@BeforeClass
public static void setup() {
    yarnConfiguration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    yarnConfiguration.setInt(YarnConfiguration.NM_PMEM_MB, 768);
    yarnConfiguration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
    yarnConfiguration.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo");
    startYARNWithConfig(yarnConfiguration);
}

From source file:org.apache.flink.yarn.YARNSessionFIFOSecuredITCase.java

License:Apache License

@BeforeClass
public static void setup() {

    LOG.info("starting secure cluster environment for testing");

    yarnConfiguration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    yarnConfiguration.setInt(YarnConfiguration.NM_PMEM_MB, 768);
    yarnConfiguration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
    yarnConfiguration.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo-secured");

    SecureTestEnvironment.prepare(tmp);/*from w  ww  .  j a va 2  s  .  c  om*/

    populateYarnSecureConfigurations(yarnConfiguration, SecureTestEnvironment.getHadoopServicePrincipal(),
            SecureTestEnvironment.getTestKeytab());

    Configuration flinkConfig = new Configuration();
    flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, SecureTestEnvironment.getTestKeytab());
    flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL,
            SecureTestEnvironment.getHadoopServicePrincipal());

    SecurityUtils.SecurityConfiguration ctx = new SecurityUtils.SecurityConfiguration(flinkConfig,
            yarnConfiguration);
    try {
        TestingSecurityContext.install(ctx, SecureTestEnvironment.getClientSecurityConfigurationMap());

        SecurityUtils.getInstalledContext().runSecured(new Callable<Object>() {
            @Override
            public Integer call() {
                startYARNSecureMode(yarnConfiguration, SecureTestEnvironment.getHadoopServicePrincipal(),
                        SecureTestEnvironment.getTestKeytab());
                return null;
            }
        });

    } catch (Exception e) {
        throw new RuntimeException("Exception occurred while setting up secure test context. Reason: {}", e);
    }

}

From source file:org.apache.hive.jdbc.TestSchedulerQueue.java

License:Apache License

@Before
public void setUp() throws Exception {
    DriverManager.setLoginTimeout(0);
    miniHS2 = new MiniHS2(conf, MiniClusterType.MR);
    miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false");
    miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "true");
    miniHS2.setConfProperty(YarnConfiguration.RM_SCHEDULER,
            "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
    miniHS2.start(new HashMap<String, String>());
    HiveTestSimpleGroupMapping.primaryTag = "";
}

From source file:org.apache.hive.jdbc.TestSchedulerQueue.java

License:Apache License

/**
 * Verify:/*from  w  w w . j a va 2  s  . co  m*/
 *  Test is running with MR2 and queue mapping defaults are set.
 *  Queue mapping is set for the connected user.
 *
 * @throws Exception
 */
@Test
public void testFairSchedulerQueueMapping() throws Exception {
    hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar");
    verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false");
    verifyProperty("mapreduce.framework.name", "yarn");
    verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "true");
    verifyProperty(YarnConfiguration.RM_SCHEDULER,
            "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
    verifyProperty("mapreduce.job.queuename", "root.user1");
}

From source file:org.apache.metron.integration.components.YarnComponent.java

License:Apache License

@Override
public void start() throws UnableToStartException {
    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.set("yarn.log.dir", "target");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);

    try {/*  w w  w. ja  v a 2  s. c o m*/
        yarnCluster = new MiniYARNCluster(testName, 1, NUM_NMS, 1, 1, true);
        yarnCluster.init(conf);

        yarnCluster.start();

        waitForNMsToRegister();

        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
        //write the document to a buffer (not directly to the file, as that
        //can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        //write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
        FileContext fsContext = FileContext.getLocalFSFileContext();
        fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
        }
    } catch (Exception e) {
        throw new UnableToStartException("Exception setting up yarn cluster", e);
    }
}

From source file:org.springframework.yarn.test.TestCluster.java

License:Apache License

@BeforeClass
public static void setup() throws InterruptedException, IOException, URISyntaxException {

    LOG.info("Starting up YARN cluster");
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    if (yarnCluster == null) {
        yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, 1, 1);
        yarnCluster.init(conf);//from   w w w  .  j a v a 2s . c o m
        yarnCluster.start();
    }
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
        LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
    }
}

From source file:org.testifyproject.resource.yarn.MiniYarnResource.java

License:Apache License

@Override
public YarnConfiguration configure(TestContext testContext, LocalResource localResource,
        PropertiesReader configReader) {
    String testName = testContext.getName();
    String logDirectory = fileSystemUtil.createPath("target", "yarn", testName);

    YarnConfiguration configuration = new YarnConfiguration();
    configuration.set(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR, logDirectory);
    configuration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    configuration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);

    return configuration;
}

Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER

Introduction

Prototype

Document

Usage