Example usage for org.apache.hadoop.yarn.client.api YarnClient getApplicationReport

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.client.api YarnClient getApplicationReport.

Prototype

public abstract ApplicationReport getApplicationReport(ApplicationId appId) throws YarnException, IOException;

Source Link

Document

Get a report of the given Application.

Usage

From source file:com.cloudera.llama.am.yarn.YarnRMConnector.java

License:Apache License

private ApplicationReport _monitorAppState(YarnClient rmClient, ApplicationId appId,
        Set<YarnApplicationState> states, boolean calledFromStopped) throws LlamaException {
    String action = calledFromStopped ? "stopping" : "starting";
    try {//from  w w w.j  ava 2  s. com
        long timeout = getConf().getLong(APP_MONITOR_TIMEOUT_KEY, APP_MONITOR_TIMEOUT_DEFAULT);

        long polling = getConf().getLong(APP_MONITOR_POLLING_KEY, APP_MONITOR_POLLING_DEFAULT);

        long start = System.currentTimeMillis();
        ApplicationReport report = rmClient.getApplicationReport(appId);
        while (!states.contains(report.getYarnApplicationState())) {
            if (System.currentTimeMillis() - start > timeout) {
                throw new LlamaException(ErrorCode.AM_TIMED_OUT_STARTING_STOPPING, appId, timeout,
                        report.getYarnApplicationState(), states, action);
            }
            Thread.sleep(polling);
            report = rmClient.getApplicationReport(appId);
        }
        return report;
    } catch (Exception ex) {
        if (!calledFromStopped) {
            _stop(FinalApplicationStatus.FAILED, "Could not start, error: " + ex, true);
        }
        throw new LlamaException(ex, ErrorCode.AM_FAILED_WHILE_STARTING_STOPPING, appId, action);
    }
}

---

From source file:com.datatorrent.stram.client.StramAgent.java

License:Apache License

private StramWebServicesInfo retrieveWebServicesInfo(String appId) {
    YarnClient yarnClient = YarnClient.createYarnClient();
    String url;/*from  w  w w . ja va2  s . c om*/
    try {
        yarnClient.init(conf);
        yarnClient.start();
        ApplicationReport ar = yarnClient.getApplicationReport(ConverterUtils.toApplicationId(appId));
        String trackingUrl = ar.getTrackingUrl();
        if (!trackingUrl.startsWith("http://") && !trackingUrl.startsWith("https://")) {
            url = "http://" + trackingUrl;
        } else {
            url = trackingUrl;
        }
        if (StringUtils.isBlank(url)) {
            LOG.error("Cannot get tracking url from YARN");
            return null;
        }
        if (url.endsWith("/")) {
            url = url.substring(0, url.length() - 1);
        }
        url += WebServices.PATH;
    } catch (Exception ex) {
        //LOG.error("Caught exception when retrieving web services info", ex);
        return null;
    } finally {
        yarnClient.stop();
    }

    WebServicesClient webServicesClient = new WebServicesClient();
    try {
        JSONObject response;
        String secToken = null;
        ClientResponse clientResponse;
        int i = 0;
        while (true) {
            LOG.debug("Accessing url {}", url);
            clientResponse = webServicesClient.process(url, ClientResponse.class,
                    new WebServicesClient.GetWebServicesHandler<ClientResponse>());
            String val = clientResponse.getHeaders().getFirst("Refresh");
            if (val == null) {
                break;
            }
            int index = val.indexOf("url=");
            if (index < 0) {
                break;
            }
            url = val.substring(index + 4);
            if (i++ > MAX_REDIRECTS) {
                LOG.error("Cannot get web service info -- exceeded the max number of redirects");
                return null;
            }
        }

        if (!UserGroupInformation.isSecurityEnabled()) {
            response = new JSONObject(clientResponse.getEntity(String.class));
        } else {
            if (UserGroupInformation.isSecurityEnabled()) {
                for (NewCookie nc : clientResponse.getCookies()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Cookie " + nc.getName() + " " + nc.getValue());
                    }
                    if (nc.getName().equals(StramWSFilter.CLIENT_COOKIE)) {
                        secToken = nc.getValue();
                    }
                }
            }
            response = new JSONObject(clientResponse.getEntity(String.class));
        }
        String version = response.getString("version");
        response = webServicesClient.process(url + "/" + version + "/stram/info", JSONObject.class,
                new WebServicesClient.GetWebServicesHandler<JSONObject>());
        String appMasterUrl = response.getString("appMasterTrackingUrl");
        String appPath = response.getString("appPath");
        String user = response.getString("user");
        JSONObject permissionsInfo = null;
        FSDataInputStream is = null;
        try {
            is = fileSystem.open(new Path(appPath, "permissions.json"));
            permissionsInfo = new JSONObject(IOUtils.toString(is));
        } catch (JSONException ex) {
            LOG.error("Error reading from the permissions info. Ignoring", ex);
        } catch (IOException ex) {
            // ignore
        } finally {
            IOUtils.closeQuietly(is);
        }
        return new StramWebServicesInfo(appMasterUrl, version, appPath, user, secToken, permissionsInfo);
    } catch (Exception ex) {
        LOG.debug("Caught exception when retrieving web service info for app " + appId, ex);
        return null;
    }
}

---

From source file:com.github.sakserv.minicluster.simpleyarnapp.Client.java

License:Apache License

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);// ww  w  .  j  av a  2s  .c  o  m
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M"
            + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " "
            + resourceManagerAddress + " " + resourceManagerHostname + " " + resourceManagerSchedulerAddress
            + " " + resourceManagerResourceTrackerAddress + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
            + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Application " + appId + " finished with" + " state " + appState + " at "
            + appReport.getFinishTime());

}

---

From source file:com.gpiskas.yarn.Client.java

License:Open Source License

private void run() throws Exception {
    conf = new YarnConfiguration();

    // Create Yarn Client
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);/*w w w . j ava2 s  .c o m*/
    client.start();

    // Create Application
    YarnClientApplication app = client.createApplication();

    // Create AM Container
    ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class);
    amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M"
            + " com.gpiskas.yarn.AppMaster" + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
            + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

    // Set AM jar
    LocalResource jar = Records.newRecord(LocalResource.class);
    Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf);
    amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar));

    // Set AM CLASSPATH
    Map<String, String> env = new HashMap<String, String>();
    Utils.setUpEnv(env, conf);
    amCLC.setEnvironment(env);

    // Set AM resources
    Resource res = Records.newRecord(Resource.class);
    res.setMemory(256);
    res.setVirtualCores(1);

    // Create ApplicationSubmissionContext
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("YARNAPP");
    appContext.setQueue("default");
    appContext.setAMContainerSpec(amCLC);
    appContext.setResource(res);

    // Submit Application
    ApplicationId id = appContext.getApplicationId();
    System.out.println("Client: Submitting " + id);
    client.submitApplication(appContext);

    ApplicationReport appReport = client.getApplicationReport(id);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(1000);
        appReport = client.getApplicationReport(id);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Client: Finished " + id + " with state " + appState);
}

---

From source file:com.hazelcast.yarn.HazelcastYarnClient.java

License:Open Source License

private void init() throws Exception {
    YarnClient yarnClient = startYarnClient(this.conf);

    ApplicationId appId = submitApplication(yarnClient);
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();

    appState = process(yarnClient, appId, appState);

    LOG.log(Level.INFO, "Application {0} is {1}.", new Object[] { appId, appState });
}

---

From source file:com.hazelcast.yarn.HazelcastYarnClient.java

License:Open Source License

private YarnApplicationState process(YarnClient yarnClient, ApplicationId appId, YarnApplicationState appState)
        throws InterruptedException, YarnException, IOException {
    ApplicationReport appReport;/*ww w  .  ja  va  2s .co m*/

    while (appState == YarnApplicationState.NEW || appState == YarnApplicationState.NEW_SAVING
            || appState == YarnApplicationState.SUBMITTED || appState == YarnApplicationState.ACCEPTED) {
        TimeUnit.SECONDS.sleep(1L);
        appReport = yarnClient.getApplicationReport(appId);

        if ((appState != YarnApplicationState.ACCEPTED)
                && (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED)) {
            LOG.log(Level.INFO, "Application {0} is ACCEPTED.", appId);
        }

        appState = appReport.getYarnApplicationState();
    }

    return appState;
}

---

From source file:com.ibm.bi.dml.yarn.DMLYarnClient.java

License:Open Source License

/**
 * Method to launch the dml yarn app master and execute the given dml script
 * with the given configuration and jar file.
 * //from ww w  .j  a  v  a  2 s. c  o m
 * NOTE: on launching the yarn app master, we do not explicitly probe if we
 *     are running on a yarn or MR1 cluster. In case of MR1, already the class 
 *     YarnConfiguration will not be found and raise a classnotfound. In case of any 
 *     exception we fall back to run CP directly in the client process.
 * 
 * @return true if dml program successfully executed as yarn app master
 * @throws IOException 
 */
protected boolean launchDMLYarnAppmaster() throws IOException, DMLScriptException {
    boolean ret = false;
    String hdfsWD = null;

    try {
        Timing time = new Timing(true);

        // load yarn configuration
        YarnConfiguration yconf = new YarnConfiguration();

        // create yarn client
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(yconf);
        yarnClient.start();

        // create application and get the ApplicationID
        YarnClientApplication app = yarnClient.createApplication();
        ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
        ApplicationId appId = appContext.getApplicationId();
        LOG.debug("Created application (applicationID: " + appId + ")");

        // prepare hdfs working directory via ApplicationID
        // copy script, config, jar file to hdfs
        hdfsWD = DMLAppMasterUtils.constructHDFSWorkingDir(_dmlConfig, appId);
        copyResourcesToHdfsWorkingDir(yconf, hdfsWD);

        //construct command line argument
        String command = constructAMCommand(_args, _dmlConfig);
        LOG.debug("Constructed application master command: \n" + command);

        // set up the container launch context for the application master
        ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
        amContainer.setCommands(Collections.singletonList(command));
        amContainer.setLocalResources(constructLocalResourceMap(yconf));
        amContainer.setEnvironment(constructEnvionmentMap(yconf));

        // Set up resource type requirements for ApplicationMaster
        int memHeap = _dmlConfig.getIntValue(DMLConfig.YARN_APPMASTERMEM);
        int memAlloc = (int) computeMemoryAllocation(memHeap);
        Resource capability = Records.newRecord(Resource.class);
        capability.setMemory(memAlloc);
        capability.setVirtualCores(NUM_CORES);
        LOG.debug("Requested application resources: memory=" + memAlloc + ", vcores=" + NUM_CORES);

        // Finally, set-up ApplicationSubmissionContext for the application
        String qname = _dmlConfig.getTextValue(DMLConfig.YARN_APPQUEUE);
        appContext.setApplicationName(APPMASTER_NAME); // application name
        appContext.setAMContainerSpec(amContainer);
        appContext.setResource(capability);
        appContext.setQueue(qname); // queue
        LOG.debug("Configured application meta data: name=" + APPMASTER_NAME + ", queue=" + qname);

        // submit application (non-blocking)
        yarnClient.submitApplication(appContext);

        // Check application status periodically (and output web ui address)
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);
        LOG.info("Application tracking-URL: " + appReport.getTrackingUrl());
        YarnApplicationState appState = appReport.getYarnApplicationState();
        YarnApplicationState oldState = appState;
        LOG.info("Application state: " + appState);
        while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
                && appState != YarnApplicationState.FAILED) {
            Thread.sleep(APP_STATE_INTERVAL); //wait for 200ms
            appReport = yarnClient.getApplicationReport(appId);
            appState = appReport.getYarnApplicationState();
            if (appState != oldState) {
                oldState = appState;
                LOG.info("Application state: " + appState);
            }
        }
        //check final status (failed or succeeded)
        FinalApplicationStatus finalState = appReport.getFinalApplicationStatus();
        LOG.info("Application final status: " + finalState);

        //show application and total runtime
        double appRuntime = (double) (appReport.getFinishTime() - appReport.getStartTime()) / 1000;
        LOG.info("Application runtime: " + appRuntime + " sec.");
        LOG.info("Total runtime: " + String.format("%.3f", time.stop() / 1000) + " sec.");

        //raised script-level error in case of failed final status
        if (finalState != FinalApplicationStatus.SUCCEEDED) {
            //propagate script-level stop call message
            String stop_msg = readMessageToHDFSWorkingDir(_dmlConfig, yconf, appId);
            if (stop_msg != null)
                throw new DMLScriptException(stop_msg);

            //generic failure message
            throw new DMLRuntimeException(
                    "DML yarn app master finished with final status: " + finalState + ".");
        }

        ret = true;
    } catch (DMLScriptException ex) {
        //rethrow DMLScriptException to propagate stop call
        throw ex;
    } catch (Exception ex) {
        LOG.error("Failed to run DML yarn app master.", ex);
        ret = false;
    } finally {
        //cleanup working directory
        if (hdfsWD != null)
            MapReduceTool.deleteFileIfExistOnHDFS(hdfsWD);
    }

    return ret;
}

---

From source file:com.resa.yarn.Client.java

License:Open Source License

private void run() throws Exception {
    conf = new YarnConfiguration();

    // Create Yarn Client
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);/*  w w  w .j  a  va 2 s.  com*/
    client.start();

    // Create Application
    YarnClientApplication app = client.createApplication();

    // Create AM Container
    ContainerLaunchContext amCLC = Records.newRecord(ContainerLaunchContext.class);
    amCLC.setCommands(Collections.singletonList("$JAVA_HOME/bin/java" + " -Xmx256M" + " com.resa.yarn.AppMaster"
            + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>"
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

    // Set AM jar
    LocalResource jar = Records.newRecord(LocalResource.class);
    Utils.setUpLocalResource(Utils.YARNAPP_JAR_PATH, jar, conf);
    amCLC.setLocalResources(Collections.singletonMap(Utils.YARNAPP_JAR_NAME, jar));

    // Set AM CLASSPATH
    Map<String, String> env = new HashMap<String, String>();
    Utils.setUpEnv(env, conf);
    amCLC.setEnvironment(env);

    // Set AM resources
    Resource res = Records.newRecord(Resource.class);
    res.setMemory(256);
    res.setVirtualCores(1);

    // Create ApplicationSubmissionContext
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName("YARNAPP");
    appContext.setQueue("default");
    appContext.setAMContainerSpec(amCLC);
    appContext.setResource(res);

    // Submit Application
    ApplicationId id = appContext.getApplicationId();
    System.out.println("Client: Submitting " + id);
    client.submitApplication(appContext);

    ApplicationReport appReport = client.getApplicationReport(id);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(1000);
        appReport = client.getApplicationReport(id);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Client: Finished " + id + " with state " + appState);
}

---

From source file:edu.uci.ics.asterix.aoya.AsterixYARNClient.java

License:Apache License

/**
 * Asks YARN to kill a given application by appId
 * @param appId The application to kill.
 * @param yarnClient The YARN client object that is connected to the RM.
 * @throws YarnException/*ww w  . j  a  v  a2 s .  c o m*/
 * @throws IOException
 */

public static void killApplication(ApplicationId appId, YarnClient yarnClient)
        throws YarnException, IOException {
    if (appId == null) {
        throw new YarnException("No Application given to kill");
    }
    if (yarnClient.isInState(STATE.INITED)) {
        yarnClient.start();
    }
    YarnApplicationState st;
    ApplicationReport rep = yarnClient.getApplicationReport(appId);
    st = rep.getYarnApplicationState();
    if (st == YarnApplicationState.FINISHED || st == YarnApplicationState.KILLED
            || st == YarnApplicationState.FAILED) {
        LOG.info("Application " + appId + " already exited.");
        return;
    }
    LOG.info("Killing applicaiton with ID: " + appId);
    yarnClient.killApplication(appId);

}

---

From source file:edu.uci.ics.asterix.aoya.Utils.java

License:Apache License

public static boolean waitForLiveness(ApplicationId appId, boolean probe, boolean print, String message,
        YarnClient yarnClient, String instanceName, Configuration conf, int port) throws YarnException {
    ApplicationReport report;/*from   w  w  w  .j  av  a2  s  . c o  m*/
    try {
        report = yarnClient.getApplicationReport(appId);
    } catch (IOException e) {
        throw new YarnException(e);
    }
    YarnApplicationState st = report.getYarnApplicationState();
    for (int i = 0; i < 120; i++) {
        if (st != YarnApplicationState.RUNNING) {
            try {
                report = yarnClient.getApplicationReport(appId);
                st = report.getYarnApplicationState();
                if (print) {
                    System.out.print(message + Utils.makeDots(i) + "\r");
                }
                Thread.sleep(1000);
            } catch (InterruptedException e1) {
                Thread.currentThread().interrupt();
            } catch (IOException e1) {
                throw new YarnException(e1);
            }
            if (st == YarnApplicationState.FAILED || st == YarnApplicationState.FINISHED
                    || st == YarnApplicationState.KILLED) {
                return false;
            }
        }
        if (probe) {
            String host;
            host = getCCHostname(instanceName, conf);
            try {
                for (int j = 0; j < 60; j++) {
                    if (!Utils.probeLiveness(host, port)) {
                        try {
                            if (print) {
                                System.out.print(message + Utils.makeDots(i) + "\r");
                            }
                            Thread.sleep(1000);
                        } catch (InterruptedException e2) {
                            Thread.currentThread().interrupt();
                        }
                    } else {
                        if (print) {
                            System.out.println("");
                        }
                        return true;
                    }
                }
            } catch (IOException e1) {
                throw new YarnException(e1);
            }
        } else {
            if (print) {
                System.out.println("");
            }
            return true;
        }
    }
    if (print) {
        System.out.println("");
    }
    return false;
}

---

• «
• 1
• 2
• 3
• 4
• »