Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration()

Source Link

Usage

From source file:org.dknight.app.ApplicationMaster.java

License:Apache License

public ApplicationMaster() {
    // Set up the configuration
    conf = new YarnConfiguration();
    Path clusterConfPath = new Path("cluster-conf.xml");
    conf.addResource(clusterConfPath);//from ww w.  j  av a  2  s .  com
    LOG.info("Load the cluster configuration file from" + clusterConfPath.getName());

}

From source file:org.elasticsearch.hadoop.yarn.am.ApplicationMaster.java

License:Apache License

ApplicationMaster(Map<String, String> env) {
    this.env = env;
    cfg = new YarnConfiguration();
    if (env.containsKey(FS_URI)) {
        cfg.set(FileSystem.FS_DEFAULT_NAME_KEY, env.get(FS_URI));
    }//ww w .ja va  2  s. c om
    appConfig = new Config(PropertiesUtils.propsFromBase64String(env.get(CFG_PROPS)));
}

From source file:org.etosha.cumulusonyarn.CumulusApplicationMaster.java

License:Apache License

public CumulusApplicationMaster() throws Exception {
    conf = new YarnConfiguration();
}

From source file:org.etosha.cumulusonyarn.CumulusRDFRunner.java

License:Apache License

/**
*/
public CumulusRDFRunner() throws Exception {
    this(new YarnConfiguration());
}

From source file:org.hdl.tensorflow.yarn.appmaster.ApplicationMaster.java

License:Apache License

public ApplicationMaster() {
    super("ApplicationMaster");
    conf = new YarnConfiguration();
}

From source file:org.hdl.tensorflow.yarn.client.Client.java

License:Apache License

public Client() {
    this(new YarnConfiguration(), YarnClient.createYarnClient());
}

From source file:org.lab41.graphlab.twill.Main.java

License:Apache License

public static void main(String[] args) {
    CommandLineParser parser = new GnuParser();

    Options options = new Options();

    options.addOption("h", "help", false, "print this message");
    options.addOption("i", "instances", true, "number of instances");
    options.addOption("t", "threads", true, "number of threads");
    options.addOption("debug", false, "enable debugging");

    int instanceCount = 1;
    int virtualCores = 1;
    boolean debug = false;

    try {/*from w  ww  .ja v a2s.  co  m*/
        CommandLine line = parser.parse(options, args, true);

        if (line.hasOption("help")) {
            printHelp(options);
            System.exit(0);
        }

        if (line.hasOption("instances")) {
            String option = line.getOptionValue("instances");
            instanceCount = Integer.parseInt(option);
        }

        if (line.hasOption("threads")) {
            String option = line.getOptionValue("threads");
            virtualCores = Integer.parseInt(option);
        }

        if (line.hasOption("debug")) {
            debug = true;
        }

        args = line.getArgs();

    } catch (ParseException e) {
        System.out.println("error: " + e);
        System.exit(1);
    }

    if (args.length != 5) {
        printHelp(options);
        System.exit(1);
    }

    String zkStr = args[0];
    GraphLabRunnable.Arguments arguments = GraphLabRunnable.Arguments
            .fromArray(Arrays.copyOfRange(args, 1, args.length));

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zkStr);
    twillRunner.startAndWait();

    ResourceSpecification resources = ResourceSpecification.Builder.with().setVirtualCores(virtualCores)
            .setMemory(512, ResourceSpecification.SizeUnit.MEGA).setInstances(instanceCount).build();

    String runnableName = "GraphLabRunnable";

    TwillPreparer preparer = twillRunner.prepare(new GraphLabRunnable(), resources)
            .withArguments(runnableName, arguments.toArray())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)));

    if (debug) {
        preparer.enableDebugging(true);
    }

    final TwillController controller = preparer.start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            LOG.debug("shutting down");
            controller.stopAndWait();
            twillRunner.stopAndWait();
        }
    });

    try {
        /*
        // Try to catch the debug port.
        if (debug) {
        waitForDebugPort(controller, runnableName, 300);
        }
        */

        Services.getCompletionFuture(controller).get();
    } catch (InterruptedException | ExecutionException e) {
        e.printStackTrace();
    }

    LOG.debug("after shutting down");
}

From source file:org.springframework.yarn.configuration.ConfigurationUtils.java

License:Apache License

/**
 * Creates a new {@link Configuration} by merging the given configurations.
 * Ordering is important - the second configuration overriding values in the first.
 *
 * @param one configuration to read from. May be null.
 * @param two configuration to read from. May be null.
 * @return the result of merging the two configurations.
 *//*www. j  a v a2 s  .com*/
public static Configuration merge(Configuration one, Configuration two) {
    if (one == null) {
        if (two == null) {
            return new YarnConfiguration();
        }
        return new YarnConfiguration(two);
    }

    Configuration c = new YarnConfiguration(one);

    if (two == null) {
        return c;
    }

    for (Map.Entry<String, String> entry : two) {
        c.set(entry.getKey(), entry.getValue());
    }

    return c;
}

From source file:org.springframework.yarn.test.support.StandaloneYarnCluster.java

License:Apache License

@Override
public void start() throws IOException {
    log.info("Checking if cluster=" + clusterName + " needs to be started");
    synchronized (this.startupShutdownMonitor) {
        if (started) {
            return;
        }//from  w w  w  . j a  v a  2 s .co m
        log.info("Starting cluster=" + clusterName);
        configuration = new YarnConfiguration();
        configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs");

        dfsCluster = new MiniDFSCluster.Builder(configuration).numDataNodes(nodes).build();

        yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1);
        yarnCluster.init(configuration);
        yarnCluster.start();

        log.info("Started cluster=" + clusterName);
        started = true;
    }
}

From source file:org.testifyproject.resource.yarn.MiniYarnResource.java

License:Apache License

@Override
public YarnConfiguration configure(TestContext testContext, LocalResource localResource,
        PropertiesReader configReader) {
    String testName = testContext.getName();
    String logDirectory = fileSystemUtil.createPath("target", "yarn", testName);

    YarnConfiguration configuration = new YarnConfiguration();
    configuration.set(YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR, logDirectory);
    configuration.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
    configuration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);

    return configuration;
}