Example usage for org.apache.hadoop.yarn.conf YarnConfiguration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration addResource.

Prototype

public void addResource(String name) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.intropro.prairie.unit.yarn.YarnUnit.java

License:Apache License

@Override
protected YarnConfiguration gatherConfigs() {
    YarnConfiguration yarnConfigs = new YarnConfiguration(super.gatherConfigs());
    yarnConfigs.set("fs.defaultFS", hdfsUnit.getNamenode());
    yarnConfigs.set("mapreduce.task.tmp.dir", getTmpDir().toString());
    String user = System.getProperty("user.name");
    yarnConfigs.set("hadoop.proxyuser." + user + ".hosts", "*");
    yarnConfigs.set("hadoop.proxyuser." + user + ".groups", "*");
    yarnConfigs.set("yarn.nodemanager.admin-env", "PATH=$PATH:" + cmdUnit.getPath());
    yarnConfigs.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    yarnConfigs.addResource("mapred-site.prairie.xml");
    yarnConfigs.addResource("yarn-site.prairie.xml");
    return yarnConfigs;
}

---

From source file:io.fluo.cluster.OracleApp.java

License:Apache License

public static void main(String[] args) throws ConfigurationException, Exception {

    OracleAppOptions options = new OracleAppOptions();
    JCommander jcommand = new JCommander(options, args);

    if (options.displayHelp()) {
        jcommand.usage();/*w w  w .j  a  va  2  s. co  m*/
        System.exit(-1);
    }

    Logging.init("oracle", options.getFluoHome() + "/conf", "STDOUT");

    File configFile = new File(options.getFluoHome() + "/conf/fluo.properties");
    FluoConfiguration config = new FluoConfiguration(configFile);
    if (!config.hasRequiredOracleProps()) {
        log.error("fluo.properties is missing required properties for oracle");
        System.exit(-1);
    }
    Environment env = new Environment(config);

    YarnConfiguration yarnConfig = new YarnConfiguration();
    yarnConfig.addResource(new Path(options.getHadoopPrefix() + "/etc/hadoop/core-site.xml"));
    yarnConfig.addResource(new Path(options.getHadoopPrefix() + "/etc/hadoop/yarn-site.xml"));

    TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getZookeepers());
    twillRunner.startAndWait();

    TwillPreparer preparer = twillRunner.prepare(new OracleApp(options, config));

    TwillController controller = preparer.start();
    controller.start();

    while (controller.isRunning() == false)
        Thread.sleep(2000);

    env.close();
    System.exit(0);
}

---

From source file:io.fluo.cluster.runner.YarnAppRunner.java

License:Apache License

private synchronized TwillRunnerService getTwillRunner() {
    if (twillRunner == null) {
        YarnConfiguration yarnConfig = new YarnConfiguration();
        yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml"));
        yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml"));

        twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL);
        twillRunner.startAndWait();/*www .  j  a va 2s.com*/

        // sleep to give twill time to retrieve state from zookeeper
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            throw new IllegalStateException(e);
        }
    }
    return twillRunner;
}

---

From source file:io.fluo.cluster.WorkerApp.java

License:Apache License

public static void main(String[] args) throws ConfigurationException, Exception {

    AppOptions options = new AppOptions();
    JCommander jcommand = new JCommander(options, args);

    if (options.displayHelp()) {
        jcommand.usage();//w  ww  . j a v  a 2s .  c o  m
        System.exit(-1);
    }

    Logging.init("worker", options.getFluoHome() + "/conf", "STDOUT");

    File configFile = new File(options.getFluoHome() + "/conf/fluo.properties");
    FluoConfiguration config = new FluoConfiguration(configFile);
    if (!config.hasRequiredWorkerProps()) {
        log.error("fluo.properties is missing required properties for worker");
        System.exit(-1);
    }
    Environment env = new Environment(config);

    YarnConfiguration yarnConfig = new YarnConfiguration();
    yarnConfig.addResource(new Path(options.getHadoopPrefix() + "/etc/hadoop/core-site.xml"));
    yarnConfig.addResource(new Path(options.getHadoopPrefix() + "/etc/hadoop/yarn-site.xml"));

    TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, env.getZookeepers());
    twillRunner.startAndWait();

    TwillPreparer preparer = twillRunner.prepare(new WorkerApp(options, config));

    // Add any observer jars found in lib observers
    File observerDir = new File(options.getFluoHome() + "/lib/observers");
    for (File f : observerDir.listFiles()) {
        String jarPath = "file:" + f.getCanonicalPath();
        log.debug("Adding observer jar " + jarPath + " to YARN app");
        preparer.withResources(new URI(jarPath));
    }

    TwillController controller = preparer.start();
    controller.start();

    while (controller.isRunning() == false) {
        Thread.sleep(2000);
    }
    env.close();
    System.exit(0);
}

---

From source file:org.apache.fluo.cluster.runner.YarnAppRunner.java

License:Apache License

private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) {
    if (!twillRunners.containsKey(config.getApplicationName())) {
        YarnConfiguration yarnConfig = new YarnConfiguration();
        yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml"));
        yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml"));

        TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig,
                config.getAppZookeepers() + ZookeeperPath.TWILL);
        twillRunner.start();//  w  w  w .j  av  a 2s.  co m

        twillRunners.put(config.getApplicationName(), twillRunner);

        // sleep to give twill time to retrieve state from zookeeper
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            throw new IllegalStateException(e);
        }
    }
    return twillRunners.get(config.getApplicationName());
}

---