Example usage for org.apache.hadoop.yarn.conf YarnConfiguration NODE_LABELS_ENABLED

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration NODE_LABELS_ENABLED.

Prototype

String NODE_LABELS_ENABLED

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration NODE_LABELS_ENABLED.

Click Source Link

Document

Flag to indicate if the node labels feature enabled, by default it's disabled

Usage

From source file:io.hops.tensorflow.TestCluster.java

License:Apache License

protected void setupInternal(int numNodeManager) throws Exception {

    LOG.info("Starting up YARN cluster");

    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.set("yarn.log.dir", "target");
    conf.set("yarn.log-aggregation-enable", "true");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false);

    if (yarnCluster == null) {
        yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1);
        yarnCluster.init(conf);//from  w  w  w .ja  v a 2  s .  c o  m

        yarnCluster.start();

        conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
                MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort());

        waitForNMsToRegister();

        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
        //write the document to a buffer (not directly to the file, as that
        //can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        //write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
    }
    FileContext fsContext = FileContext.getLocalFSFileContext();
    fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
    try {
        Thread.sleep(2000);
    } catch (InterruptedException e) {
        LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
    }
}

---

From source file:org.apache.metron.integration.components.YarnComponent.java

License:Apache License

@Override
public void start() throws UnableToStartException {
    conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.set("yarn.log.dir", "target");
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);

    try {//www  .j av  a 2  s.  c  o  m
        yarnCluster = new MiniYARNCluster(testName, 1, NUM_NMS, 1, 1, true);
        yarnCluster.init(conf);

        yarnCluster.start();

        waitForNMsToRegister();

        URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
        if (url == null) {
            throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
        }
        Configuration yarnClusterConfig = yarnCluster.getConfig();
        yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent());
        //write the document to a buffer (not directly to the file, as that
        //can cause the file being written to get read -which will then fail.
        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
        yarnClusterConfig.writeXml(bytesOut);
        bytesOut.close();
        //write the bytes to the file in the classpath
        OutputStream os = new FileOutputStream(new File(url.getPath()));
        os.write(bytesOut.toByteArray());
        os.close();
        FileContext fsContext = FileContext.getLocalFSFileContext();
        fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true);
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
        }
    } catch (Exception e) {
        throw new UnableToStartException("Exception setting up yarn cluster", e);
    }
}

---