Example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration YarnConfiguration.

Prototype

public YarnConfiguration() 

Source Link

Usage

From source file:yarn.demo.InteractableYarnApplicationContainersClusterDemo.java

License:Apache License

/**
 * Ensure valid YarnConfiguration is available in the classpath, then run.
 *///from   w  w w .j a v  a2s. c o m
public static void main(String[] args) throws Exception {
    MiniClusterUtils.startMiniCluster();

    ConfigUtils.addToClasspath(new File("mini-cluster-config"));

    YarnApplication<DataProcessor> yarnApplication = YarnAssembly
            .forApplicationContainer(DemoEchoContainer.class).containerCount(4)
            .withApplicationMaster(new YarnConfiguration()).maxAttempts(2)
            .build("InteractableYarnApplicationContainersClusterDemo");

    yarnApplication.registerReplyListener(new ContainerReplyListener() {
        @Override
        public void onReply(ByteBuffer replyData) {
            byte[] replyBytes = new byte[replyData.limit()];
            replyData.rewind();
            replyData.get(replyBytes);
            replyData.rewind();
            String reply = new String(replyBytes);
            System.out.println("REPLY: " + reply);
        }
    });
    /*
     * DataProcessor essentially is a proxy over all Application Containers defined by this 
     * application and running in YARN (8 in this case).
     * It is aware of which Application Containers are available and will 
     * delegate its process(..) invocation to the first available Application Container.
     * So essentially DataProcessor is a gateway to the YARN Distributed Computing Grid.
     * 
     * Additionally you can register oz.hadoop.yarn.api.DataProcessorReplyListener with 
     * DataProcessor if interested in receiving a reply from the distributed process.
     */
    final DataProcessor dataProcessor = yarnApplication.launch();

    for (int i = 0; i < 30; i++) {
        /*
         * Actual processing will be delegated to the first available (out of 8 deployed) 
         * remote Application Containers.
         */
        dataProcessor.process(ByteBuffer.wrap(("Hello Yarn Grid! - " + i).getBytes()));
    }

    yarnApplication.shutDown();
    System.out.println("Processes completed since launch: " + dataProcessor.completedSinceStart());

    MiniClusterUtils.stoptMiniCluster();
}

---

From source file:yarn.demo.JavaBasedYarnApplicationClusterDemo.java

License:Apache License

/**
 * Before running ensure that properly configured yarn-site.xml are copied
 * into src/main/resources. You can use the yarn-site.xml from local-config
 * directory of this project. The newly checkout out project is already
 * setup for this.//  w w  w  .j ava  2 s.  co m
 * Examples for remote configurations are located in remote-config directory,
 * but you might as well use the ones from your installed cluster.
 *
 * If running in Mini-Cluster (see yarn-test-cluster project), make sure you start it
 * by executing StartMiniCluster.java first.
 */
public static void main(String[] args) throws Exception {
    MiniClusterUtils.startMiniCluster();

    ConfigUtils.addToClasspath(new File("mini-cluster-config"));

    YarnApplication<Void> yarnApplication = YarnAssembly
            .forApplicationContainer(ReverseMessageContainer.class, ByteBuffer.wrap("Hello Yarn!".getBytes()))
            .containerCount(4).withApplicationMaster(new YarnConfiguration()).maxAttempts(2)
            .build("JavaBasedYarnApplicationDemo");

    yarnApplication.launch();
    yarnApplication.awaitFinish();

    MiniClusterUtils.stoptMiniCluster();
}

---

From source file:yrun.commands.Ykill.java

License:Apache License

public static void main(String[] args) throws IOException {
    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);//from w ww . j  a v  a 2  s. c  om
    yarnClient.start();
    boolean debug = false;
    try {
        Splitter splitter = Splitter.on('_');
        for (String arg : args) {
            List<String> list = toList(splitter.split(arg));
            if (list.size() != 3) {
                System.err.println("Application Id " + arg + " is not a valid application id.");
            } else {
                String prefix = list.get(0);
                if (!prefix.equals("application")) {
                    System.err.println("Application Id " + arg + " is not a valid application id.");
                } else {
                    try {
                        long clusterTimestamp = Long.parseLong(list.get(1));
                        int id = Integer.parseInt(list.get(2));
                        ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp, id);
                        yarnClient.killApplication(applicationId);
                        System.out.println("Killed\t" + arg + "");
                    } catch (Exception e) {
                        if (debug) {
                            e.printStackTrace();
                        }
                        System.err.println("Error while trying to kill " + arg + ".");
                    }
                }
            }
        }
    } finally {
        yarnClient.stop();
        yarnClient.close();
    }
}

---

From source file:yrun.commands.Yps.java

License:Apache License

public static void main(String[] args) throws YarnException, IOException {
    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);// w w  w  .java2s .  com
    yarnClient.start();
    try {
        List<ApplicationReport> applications = yarnClient.getApplications();
        for (ApplicationReport applicationReport : applications) {
            ApplicationId applicationId = applicationReport.getApplicationId();
            String user = applicationReport.getUser();
            String queue = applicationReport.getQueue();
            String name = applicationReport.getName();
            YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();
            float progress = applicationReport.getProgress();
            System.out.printf("%s\t%s\t%s\t%s\t%s\t%f%n", toString(applicationId), user, queue, name,
                    yarnApplicationState.name(), progress);
        }
    } finally {
        yarnClient.stop();
        yarnClient.close();
    }
}

---

From source file:yrun.YarnRunner.java

License:Apache License

public static void main(final String[] args) throws IOException, InterruptedException {
    CommandLine cmd = parse(args, new PrintWriter(System.out));
    final String yarnName = cmd.getOptionValue("n");
    LOG.info("Yarn name [" + yarnName + "]");
    String resourceManagerAddress = cmd.getOptionValue("rma");
    LOG.info("Resource Manager Address [" + resourceManagerAddress + "]");
    String installPathStr = cmd.getOptionValue("p");
    LOG.info("Install Path [" + installPathStr + "]");
    final String command = StringUtils.join(" ", cmd.getOptionValues("c"));
    LOG.info("Command [" + command + "]");
    final String queue;
    if (cmd.hasOption("q")) {
        queue = cmd.getOptionValue("q");
    } else {/*from w ww .  j  a  va 2s .c  o m*/
        queue = "default";
    }
    LOG.info("Queue [" + queue + "]");
    final YarnConfiguration configuration = new YarnConfiguration();
    configuration.set(YARN_RESOURCEMANAGER_ADDRESS, resourceManagerAddress);
    LOG.info("Using resource manager [" + resourceManagerAddress + "]");

    final Path installPath = new Path(installPathStr);

    final List<Path> archivePathList = new ArrayList<Path>();
    if (cmd.hasOption("a")) {
        String[] archivePaths = cmd.getOptionValues("a");
        for (String archivePath : archivePaths) {
            archivePathList.add(new Path(archivePath));
        }
    }

    final boolean isDaemon = !cmd.hasOption("k");

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    ugi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
            YarnRunner yarnRunner = new YarnRunner(configuration, installPath, archivePathList, command,
                    yarnName, queue, isDaemon);
            yarnRunner.execute();
            return null;
        }
    });
}

---

From source file:yrun.YarnRunnerApplicationMaster.java

License:Apache License

public void run(String[] args) throws Exception {
    JsonParser parser = new JsonParser();
    JsonElement element = parser.parse(new FileReader(YarnRunner.MASTER_JSON));

    LOG.info("Json [" + element + "]");

    JsonObject jsonObject = (JsonObject) element;
    int priority = jsonObject.get("priority").getAsInt();
    int numberOfContainers = jsonObject.get("numberOfContainers").getAsInt();
    int memory = jsonObject.get("memory").getAsInt();
    int vCores = jsonObject.get("vCores").getAsInt();
    String command = jsonObject.get("command").getAsString();

    // startHttpServer();
    // InetSocketAddress address = _server.getAddress();
    // LOG.info("Http server started at [" + address + "]");

    // String appHostName = "app-host-name";
    // int appHostPort = address.getPort();
    // String appTrackingUrl = "http://" + address.getHostName() + ":" +
    // appHostPort + "/";
    // LOG.info("App Tracking Url [" + appTrackingUrl + "]");

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);/*from   w  w  w .ja  v  a  2s .c  o m*/
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    LOG.info("Register Application Master 0");
    String appHostName = "";
    int appHostPort = 0;
    String appTrackingUrl = "";
    rmClient.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
    LOG.info("Register Application Master 1");

    // Priority for worker containers - priorities are intra-application
    Priority priorityRecord = Records.newRecord(Priority.class);
    priorityRecord.setPriority(priority);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    capability.setVirtualCores(vCores);

    // Make container requests to ResourceManager
    for (int i = 0; i < numberOfContainers; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priorityRecord);
        LOG.info("Making resource request for [" + i + "]");
        rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    long lastReport = 0;

    List<Container> containers = new ArrayList<Container>();

    while (completedContainers < numberOfContainers) {
        if (lastReport + TimeUnit.SECONDS.toNanos(5) < System.nanoTime()) {
            for (Container container : containers) {
                ContainerId containerId = container.getId();
                NodeId nodeId = container.getNodeId();
                ContainerStatus containerStatus = nmClient.getContainerStatus(containerId, nodeId);
                LOG.info("NodeId [" + nodeId + "] Container Status [" + containerStatus + "]");

                // Figure out

            }
            lastReport = System.nanoTime();
        }

        AllocateResponse response = rmClient.allocate(responseId++);
        for (Container container : response.getAllocatedContainers()) {
            containers.add(container);
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(command + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
                            + "/stdout2" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));
            LOG.info("Launching container " + container.getId());
            nmClient.startContainer(container, ctx);
        }

        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            completedContainers++;
            LOG.info("Completed container " + status.getContainerId());
        }
        Thread.sleep(100);
    }
    // _server.stop(0);
    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• »