Example usage for org.apache.hadoop.yarn.conf YarnConfiguration get

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration get.

Prototype

public String get(String name, String defaultValue) 

Source Link

Document

Get the value of the name.

Usage

From source file:base.test.HelloWorld.java

License:Apache License

public static void main(String[] args) {
    if (args.length < 1) {
        System.err.println("Arguments format: <host:port of zookeeper server>");
        System.exit(1);/* w ww.j  ava  2s . c o  m*/
    }

    String zkStr = args[0];
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnConfiguration.setSocketAddr("yarn.resourcemanager.address",
            new InetSocketAddress("192.168.80.103", 8032));
    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
    twillRunner.start();

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
    List<String> applicationClassPaths = Lists.newArrayList();
    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.cloudera.kitten.appmaster.service.ResourceManagerConnectionFactory.java

License:Open Source License

@Override
public AMRMProtocol connect() {
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    return ((AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf));
}

---

From source file:com.cloudera.kitten.client.service.ApplicationsManagerConnectionFactory.java

License:Open Source License

@Override
public ClientRMProtocol connect() {
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    InetSocketAddress rmAddress = NetUtils
            .createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
    LOG.info("Connecting to ResourceManager at: " + rmAddress);
    return ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf));
}

---

From source file:com.datatorrent.stram.util.ConfigUtils.java

License:Apache License

public static String getSchemePrefix(YarnConfiguration conf) {
    if (HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(
            conf.get(YarnConfiguration.YARN_HTTP_POLICY_KEY, YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))) {
        return "https://";
    } else {// w w w. j a  va  2 s.co m
        return "http://";
    }
}

---

From source file:com.sina.dip.twill.HelloWorld.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();//from w  ww . j av  a  2  s.  com

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldArguments.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();//from ww w.j  a va  2 s  . c  o m

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            // Application arguments will be visible to all runnables
            .withApplicationArguments("--arg", "arg-app")
            // Arguments only visible to instance of hello1.
            .withArguments("hello1", "--arg1", "arg-hello1").withArguments("hello1", "--arg2", "arg-hello2")
            // Arguments only visible to instance of hello2.
            .withArguments("hello2", "--arg3", "arg-hello3").withArguments("hello2", "--arg4", "arg-hello4")
            .withArguments("hello2", "--arg5", "arg-hello5")
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldClassDependent.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/*ww  w.  j  a v  a2s . c om*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldControllingLiveApplications.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/*from www.  j av  a 2 s . co m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    ServiceDiscovered helloWorldService = controller.discoverService("HelloWorldService");

    ServiceDiscovered helloWorldService2 = controller.discoverService("HelloWorldService2");

    int count = 0;

    while (true) {
        boolean flag = true;

        Iterator<Discoverable> iterator = helloWorldService.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        iterator = helloWorldService2.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        try {
            Thread.sleep(5 * 1000);
        } catch (InterruptedException e) {
        }

        ++count;

        if (count == 10) {
            controller.changeInstances("hello1", 3);
            controller.changeInstances("hello2", 5);
        } else if (count == 20) {
            controller.changeInstances("hello1", 5);
            controller.changeInstances("hello2", 3);
        }

        if (count >= 36 && flag) {
            break;
        }
    }

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldMultipleRunnablesAnyOrder.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/*w ww. ja v a2s. c om*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldResourcesSpecification.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/* ww w .j  a va  2s  .c  o  m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner
            .prepare(new HelloWorldRunnable(),
                    ResourceSpecification.Builder.with().setVirtualCores(1).setMemory(2, SizeUnit.GIGA)
                            .setInstances(3).build())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

• «
• 1
• 2
• »