Example usage for org.apache.hadoop.yarn.client.api YarnClient close

List of usage examples for org.apache.hadoop.yarn.client.api YarnClient close

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.client.api.*
  5. YarnClient
  6. close

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api YarnClient close.

Prototype

@Override
public final void close() throws IOException

Source Link

Document

Relay to #stop()

Usage

From source file:com.flyhz.avengers.framework.application.InitEnvApplication.java

License:Apache License

private void initJar() {
    LOG.info("initJar");
    numTotalContainers = 1;/* w  ww. j  av  a 2s  . co m*/
    containerMemory = 10;
    if (numTotalContainers == 0) {
        throw new IllegalArgumentException("Cannot run distributed shell with no containers");
    }
    requestPriority = 0;
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();
    List<NodeReport> clusterNodeReports;
    try {
        clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
        LOG.info("Got all node!");
        for (NodeReport node : clusterNodeReports) {
            nodeSet.add(node.getNodeId().getHost());
        }

        LOG.info("initJar Completed setting up app master command {}", initJarCmd);
        numTotalContainers = nodeSet.size();
        for (int i = 0; i < numTotalContainers; i++) {
            ContainerRequest containerAsk = setupContainerAskForRM();
            amRMClient.addContainerRequest(containerAsk);
        }

        numRequestedContainers.set(numTotalContainers);

    } catch (YarnException e) {
        LOG.error("initJarAndClasspath", e);
    } catch (IOException e) {
        LOG.error("initJarAndClasspath", e);
    } finally {
        try {
            yarnClient.close();
        } catch (IOException e) {
        }
    }
}

From source file:com.flyhz.avengers.framework.AvengersAppMaster.java

License:Apache License

private void initJar() {
    LOG.info("initJar");
    initCommon();/*from   w ww  .j  a  va  2 s.c  o  m*/
    this.currentProcess = "initJar";
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();
    List<NodeReport> clusterNodeReports;
    try {
        clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
        LOG.info("Got all node!");
        for (NodeReport node : clusterNodeReports) {
            LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
                    + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
                    + node.getNumContainers());
            nodeSet.add(node.getNodeId().getHost());
        }
        Vector<CharSequence> vargs = new Vector<CharSequence>(30);
        String appTempDir = conf.get("hadoop.tmp.dir");
        FileSystem fs = DistributedFileSystem.get(conf);
        String hdfsJar = fs.getHomeDirectory() + "/avengers/" + this.appAttemptID.getApplicationId().getId()
                + "/AvengersAppMaster.jar";
        vargs.add("if [ ! -e " + appTempDir + "/" + this.appAttemptID.getApplicationId().getId()
                + "/AvengersAppMaster.jar" + " ];");
        vargs.add("then");
        vargs.add("/bin/mkdir -p");
        vargs.add(appTempDir + "/" + this.appAttemptID.getApplicationId().getId() + ";");
        vargs.add(Environment.HADOOP_YARN_HOME.$() + "/bin/hadoop");
        vargs.add("fs");
        vargs.add("-copyToLocal");
        vargs.add(hdfsJar);
        vargs.add(appTempDir + "/" + this.appAttemptID.getApplicationId().getId() + ";");
        vargs.add("fi");
        // Add log redirect params
        vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/Avengers.stdout");
        vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/Avengers.stderr");

        // Get final commmand
        StringBuilder command = new StringBuilder();
        for (CharSequence str : vargs) {
            command.append(str).append(" ");
        }

        initJarCmd = command.toString();
        LOG.info("initJar Completed setting up app master command {}", initJarCmd);
        numTotalContainers = nodeSet.size();
        for (int i = 0; i < numTotalContainers; i++) {
            ContainerRequest containerAsk = setupContainerAskForRM();
            amRMClient.addContainerRequest(containerAsk);
        }

        numRequestedContainers.set(numTotalContainers);

        while (!done && (numCompletedContainers.get() != numTotalContainers)) {
            try {
                Thread.sleep(200);
            } catch (InterruptedException ex) {
            }
        }
    } catch (YarnException e) {
        LOG.error("initJarAndClasspath", e);
    } catch (IOException e) {
        LOG.error("initJarAndClasspath", e);
    } finally {
        try {
            yarnClient.close();
        } catch (IOException e) {
        }
    }
}

From source file:io.hops.hopsworks.common.admin.llap.LlapClusterFacade.java

License:Open Source License

public boolean isClusterUp() {
    String llapAppID = variablesFacade.getVariableValue(Settings.VARIABLE_LLAP_APP_ID);
    if (llapAppID == null || llapAppID.isEmpty()) {
        return false;
    }//from w ww. ja  v a 2s .  c om

    ApplicationId appId = ApplicationId.fromString(llapAppID);
    YarnClient yarnClient = yarnClientService.getYarnClientSuper(settings.getConfiguration()).getYarnClient();
    ApplicationReport applicationReport = null;
    try {
        applicationReport = yarnClient.getApplicationReport(appId);
    } catch (IOException | YarnException e) {
        logger.log(Level.SEVERE,
                "Could not retrieve application state for llap cluster with appId: " + appId.toString(), e);
        return false;
    } finally {
        try {
            yarnClient.close();
        } catch (IOException ex) {
        }
    }

    YarnApplicationState appState = applicationReport.getYarnApplicationState();
    return appState == YarnApplicationState.RUNNING || appState == YarnApplicationState.SUBMITTED
            || appState == YarnApplicationState.ACCEPTED || appState == YarnApplicationState.NEW
            || appState == YarnApplicationState.NEW_SAVING;
}

From source file:io.hops.hopsworks.common.admin.llap.LlapClusterFacade.java

License:Open Source License

public List<String> getLlapHosts() {
    ArrayList<String> hosts = new ArrayList<>();

    if (!isClusterUp() || isClusterStarting()) {
        return hosts;
    }/*w ww.j  a va 2s. com*/

    // The cluster is app, so the appId exists
    String llapAppID = variablesFacade.getVariableValue(Settings.VARIABLE_LLAP_APP_ID);

    ApplicationId appId = ApplicationId.fromString(llapAppID);
    YarnClient yarnClient = yarnClientService.getYarnClientSuper(settings.getConfiguration()).getYarnClient();
    try {
        List<ApplicationAttemptReport> attempts = yarnClient.getApplicationAttempts(appId);
        ApplicationAttemptReport current = null;
        for (ApplicationAttemptReport attempt : attempts) {
            // Only if the app is running the metrics are available
            if (attempt.getYarnApplicationAttemptState() == YarnApplicationAttemptState.RUNNING) {
                current = attempt;
                break;
            }
        }

        if (current == null) {
            return hosts;
        }

        List<ContainerReport> containerReports = yarnClient.getContainers(current.getApplicationAttemptId());

        // For all the new/running containers, which are not the application master, get the host
        for (ContainerReport containerReport : containerReports) {
            // Only if the container is running the metrics are available
            if (containerReport.getContainerState() == ContainerState.RUNNING
                    && !containerReport.getContainerId().equals(current.getAMContainerId())) {
                hosts.add(containerReport.getAssignedNode().getHost());
            }
        }

    } catch (IOException | YarnException ex) {
        logger.log(Level.SEVERE, "Couldn't retrieve the containers for LLAP cluster", ex);
    } finally {
        try {
            yarnClient.close();
        } catch (IOException ex) {
        }
    }

    return hosts;
}

From source file:io.hops.hopsworks.common.admin.llap.LlapClusterLifecycle.java

License:Open Source License

@Lock(LockType.WRITE)
public boolean stopCluster() {
    String llapAppID = variablesFacade.getVariableValue(Settings.VARIABLE_LLAP_APP_ID);
    if (llapAppID == null || llapAppID.isEmpty()) {
        return false;
    }/*from   w ww.  j a  v a2s .c o  m*/

    ApplicationId appId = ApplicationId.fromString(llapAppID);
    YarnClient yarnClient = yarnClientService.getYarnClientSuper(settings.getConfiguration()).getYarnClient();
    try {
        yarnClient.killApplication(appId);
    } catch (IOException | YarnException e) {
        logger.log(Level.SEVERE, "Could not kill llap cluster with appId: " + appId.toString(), e);
        return false;
    } finally {
        try {
            yarnClient.close();
        } catch (IOException ex) {
        }
    }

    return true;
}

From source file:uk.gov.gchq.gaffer.slider.util.AppConfigGenerator.java

License:Apache License

private AvailableResources getYarnResources() throws IOException, YarnException {
    final Configuration config = new Configuration();
    final YarnClient yarn = YarnClient.createYarnClient();
    yarn.init(config);/*from   w ww. ja  v  a  2s.  c o  m*/
    yarn.start();

    // Query YARN to find out the largest container it is capable of scheduling
    final YarnClientApplication app = yarn.createApplication();
    final Resource resources = app.getNewApplicationResponse().getMaximumResourceCapability();

    // Also find out how many nodes there are in the cluster by asking for the number of registered Node Managers
    final YarnClusterMetrics metrics = yarn.getYarnClusterMetrics();

    yarn.close();

    return new AvailableResources(resources.getVirtualCores(), resources.getMemory(),
            metrics.getNumNodeManagers());
}

From source file:yrun.commands.Ykill.java

License:Apache License

public static void main(String[] args) throws IOException {
    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);/*from  w ww .ja v  a  2  s .  co m*/
    yarnClient.start();
    boolean debug = false;
    try {
        Splitter splitter = Splitter.on('_');
        for (String arg : args) {
            List<String> list = toList(splitter.split(arg));
            if (list.size() != 3) {
                System.err.println("Application Id " + arg + " is not a valid application id.");
            } else {
                String prefix = list.get(0);
                if (!prefix.equals("application")) {
                    System.err.println("Application Id " + arg + " is not a valid application id.");
                } else {
                    try {
                        long clusterTimestamp = Long.parseLong(list.get(1));
                        int id = Integer.parseInt(list.get(2));
                        ApplicationId applicationId = ApplicationId.newInstance(clusterTimestamp, id);
                        yarnClient.killApplication(applicationId);
                        System.out.println("Killed\t" + arg + "");
                    } catch (Exception e) {
                        if (debug) {
                            e.printStackTrace();
                        }
                        System.err.println("Error while trying to kill " + arg + ".");
                    }
                }
            }
        }
    } finally {
        yarnClient.stop();
        yarnClient.close();
    }
}

From source file:yrun.commands.Yps.java

License:Apache License

public static void main(String[] args) throws YarnException, IOException {
    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);/*from ww  w. j av a2 s . c  o  m*/
    yarnClient.start();
    try {
        List<ApplicationReport> applications = yarnClient.getApplications();
        for (ApplicationReport applicationReport : applications) {
            ApplicationId applicationId = applicationReport.getApplicationId();
            String user = applicationReport.getUser();
            String queue = applicationReport.getQueue();
            String name = applicationReport.getName();
            YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();
            float progress = applicationReport.getProgress();
            System.out.printf("%s\t%s\t%s\t%s\t%s\t%f%n", toString(applicationId), user, queue, name,
                    yarnApplicationState.name(), progress);
        }
    } finally {
        yarnClient.stop();
        yarnClient.close();
    }
}