Example usage for org.apache.hadoop.yarn.conf YarnConfiguration get

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration get.

Prototype

public String get(String name, String defaultValue) 

Source Link

Document

Get the value of the name.

Usage

From source file:com.sina.dip.twill.HelloWorldServiceDiscovery.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();//w ww . j a v a  2  s.c o m

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    ServiceDiscovered helloWorldService = controller.discoverService("HelloWorldService");

    ServiceDiscovered helloWorldService2 = controller.discoverService("HelloWorldService2");

    int count = 0;

    while (true) {
        boolean flag = true;

        Iterator<Discoverable> iterator = helloWorldService.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        iterator = helloWorldService2.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        try {
            Thread.sleep(5 * 1000);
        } catch (InterruptedException e) {
        }

        if (++count >= 36 && flag) {
            break;
        }
    }

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:edu.uci.ics.hyracks.yarn.common.protocols.amrm.AMRMConnection.java

License:Apache License

public AMRMConnection(YarnConfiguration config) {
    this.config = config;
    Map<String, String> envs = System.getenv();
    String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
    if (containerIdString == null) {
        throw new IllegalArgumentException("ContainerId not set in the environment");
    }/*from w  ww .  ja  v  a  2  s  .c  o m*/
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    appAttemptId = containerId.getApplicationAttemptId();
    InetSocketAddress rmAddress = NetUtils.createSocketAddr(
            config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
    YarnRPC rpc = YarnRPC.create(config);

    amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config);
}

---

From source file:edu.uci.ics.hyracks.yarn.common.protocols.clientrm.YarnClientRMConnection.java

License:Apache License

public YarnClientRMConnection(YarnConfiguration config) {
    this.config = config;
    InetSocketAddress remoteAddress = NetUtils
            .createSocketAddr(config.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    Configuration appsManagerServerConf = new Configuration(config);
    appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER,
            ClientRMSecurityInfo.class, SecurityInfo.class);
    YarnRPC rpc = YarnRPC.create(appsManagerServerConf);
    crmp = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf));
}

---

From source file:org.apache.hama.bsp.YARNBSPJob.java

License:Apache License

public YARNBSPJob(HamaConfiguration conf) throws IOException {
    submitClient = new YARNBSPJobClient(conf);
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    this.rpc = YarnRPC.create(conf);
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    LOG.info("Connecting to ResourceManager at " + rmAddress);

    this.applicationsManager = ((ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
            rmAddress, conf));//from   w  w w .j  a va  2s  . c  om
}

---

From source file:org.apache.tez.history.ATSImportTool.java

License:Apache License

static boolean hasHttpsPolicy(Configuration conf) {
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    return (HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(
            yarnConf.get(YarnConfiguration.YARN_HTTP_POLICY_KEY, YarnConfiguration.YARN_HTTP_POLICY_DEFAULT)));
}

---

From source file:org.apache.twill.example.yarn.HelloWorld.java

License:Apache License

public static void main(String[] args) {
    if (args.length < 1) {
        System.err.println("Arguments format: <host:port of zookeeper server>");
        System.exit(1);/*from  ww w.  j a  va2s  .  co  m*/
    }

    String zkStr = args[0];
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
    twillRunner.start();

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
    List<String> applicationClassPaths = Lists.newArrayList();
    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java

License:Apache License

public AMRMProtocol getAMResourceManager() {
    if (amResourceManager != null)
        return amResourceManager;

    LOG.debug("Using configuration: " + conf);

    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    YarnRPC rpc = YarnRPC.create(yarnConf);
    InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));

    LOG.info("Connecting to the resource manager (scheduling) at " + rmAddress);
    amResourceManager = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf);

    return amResourceManager;
}

---

From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java

License:Apache License

public ClientRMProtocol getClientResourceManager() {
    if (clientResourceManager != null)
        return clientResourceManager;

    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    YarnRPC rpc = YarnRPC.create(yarnConf);
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));

    LOG.info("Connecting to the resource manager (client) at " + rmAddress);

    clientResourceManager = (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf);

    return clientResourceManager;
}

---

From source file:org.huahinframework.manager.rest.service.ApplicationService.java

License:Apache License

/**
 * init/*from   www . j a  v  a  2  s. c  o m*/
 */
public void init() {
    YarnRPC rpc = YarnRPC.create(getJobConf());
    YarnConfiguration yarnConf = new YarnConfiguration(getJobConf());
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    applicationsManager = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, yarnConf));
    recordFactory = RecordFactoryProvider.getRecordFactory(yarnConf);
}

---

• «
• 1
• 2
• »