Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration() 

Source Link

Usage

From source file:com.resa.yarn.Client.java

License:Open Source License

private void run() throws Exception {
    conf = new YarnConfiguration();

    // Create Yarn Client
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);//from  w w  w  .  ja v a  2s . co  m
    client.start();

    // Create Application
    YarnClientApplication app = client.createApplication();

    // Create AM Container
    ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class);
    amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.resa.yarn.AppMaster"
            + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>"
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

    // Set AM jar
    LocalResource jar = Records.newRecord(LocalResource.class);
    Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf);
    amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar));

    // Set AM CLASSPATH
    Map<String, String> env = new HashMap<String, String>();
    Utils.setUpEnv(env, conf);
    amCLC.setEnvironment(env);

    // Set AM resources
    Resource res = Records.newRecord(Resource.class);
    res.setMemory(256);
    res.setVirtualCores(1);

    // Create ApplicationSubmissionContext
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("YARNAPP");
    appContext.setQueue("default");
    appContext.setAMContainerSpec(amCLC);
    appContext.setResource(res);

    // Submit Application
    ApplicationId id = appContext.getApplicationId();
    System.out.println("Client: Submitting " + id);
    client.submitApplication(appContext);

    ApplicationReport appReport = client.getApplicationReport(id);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(1000);
        appReport = client.getApplicationReport(id);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Client: Finished " + id + " with state " + appState);
}

---

From source file:com.scistor.dshell.ScistorApplicationMaster.java

License:Apache License

public ScistorApplicationMaster() {
    // Set up the configuration
    conf = new YarnConfiguration();
}

---

From source file:com.scistor.dshell.ScistorClient.java

License:Apache License

/**
 * 
 */
public ScistorClient() throws Exception {
    this(new YarnConfiguration());
}

---

From source file:com.sina.dip.twill.HelloWorld.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();//from w ww.  ja  va  2s  .com

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldArguments.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();//ww  w  .j a va2 s. c o  m

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            // Application arguments will be visible to all runnables
            .withApplicationArguments("--arg", "arg-app")
            // Arguments only visible to instance of hello1.
            .withArguments("hello1", "--arg1", "arg-hello1").withArguments("hello1", "--arg2", "arg-hello2")
            // Arguments only visible to instance of hello2.
            .withArguments("hello2", "--arg3", "arg-hello3").withArguments("hello2", "--arg4", "arg-hello4")
            .withArguments("hello2", "--arg5", "arg-hello5")
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldClassDependent.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();//from  ww  w  .ja v a 2 s. c om

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldControllingLiveApplications.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/*from   w w w .j  a  v a  2s .co m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    ServiceDiscovered helloWorldService = controller.discoverService("HelloWorldService");

    ServiceDiscovered helloWorldService2 = controller.discoverService("HelloWorldService2");

    int count = 0;

    while (true) {
        boolean flag = true;

        Iterator<Discoverable> iterator = helloWorldService.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        iterator = helloWorldService2.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        try {
            Thread.sleep(5 * 1000);
        } catch (InterruptedException e) {
        }

        ++count;

        if (count == 10) {
            controller.changeInstances("hello1", 3);
            controller.changeInstances("hello2", 5);
        } else if (count == 20) {
            controller.changeInstances("hello1", 5);
            controller.changeInstances("hello2", 3);
        }

        if (count >= 36 && flag) {
            break;
        }
    }

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldMultipleRunnablesAnyOrder.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/* w  w  w  . j  a  va2s .  c o  m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldResourcesSpecification.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/*w w w.j  a  v  a 2s.  c  o m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner
            .prepare(new HelloWorldRunnable(),
                    ResourceSpecification.Builder.with().setVirtualCores(1).setMemory(2, SizeUnit.GIGA)
                            .setInstances(3).build())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

From source file:com.sina.dip.twill.HelloWorldServiceDiscovery.java

License:Apache License

public static void main(String[] args) {
    String zkStr = "localhost:2181";

    YarnConfiguration yarnConfiguration = new YarnConfiguration();

    final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);

    twillRunner.start();/*  ww w. j a v  a 2  s . c o  m*/

    String yarnClasspath = yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            "/usr/lib/hadoop/*,/usr/lib/hadoop-0.20-mapreduce/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-yarn/*");

    List<String> applicationClassPaths = Lists.newArrayList();

    Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));

    final TwillController controller = twillRunner.prepare(new HelloWorldApplication())
            .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
            .withApplicationClassPaths(applicationClassPaths)
            .withBundlerClassAcceptor(new HadoopClassExcluder()).start();

    ServiceDiscovered helloWorldService = controller.discoverService("HelloWorldService");

    ServiceDiscovered helloWorldService2 = controller.discoverService("HelloWorldService2");

    int count = 0;

    while (true) {
        boolean flag = true;

        Iterator<Discoverable> iterator = helloWorldService.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        iterator = helloWorldService2.iterator();

        while (iterator.hasNext()) {
            Discoverable discoverable = iterator.next();

            System.out.println(discoverable.getName() + " : " + discoverable.getSocketAddress());

            flag = false;
        }

        try {
            Thread.sleep(5 * 1000);
        } catch (InterruptedException e) {
        }

        if (++count >= 36 && flag) {
            break;
        }
    }

    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override
        public void run() {
            try {
                Futures.getUnchecked(controller.terminate());
            } finally {
                twillRunner.stop();
            }
        }
    });

    try {
        controller.awaitTerminated();
    } catch (ExecutionException e) {
        e.printStackTrace();
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »