Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getYarnApplicationState

List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getYarnApplicationState

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ApplicationReport
  6. getYarnApplicationState

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getYarnApplicationState.

Prototype

@Public
@Stable
public abstract YarnApplicationState getYarnApplicationState();

Source Link

Document

Get the YarnApplicationState of the application.

Usage

From source file:timo.yarn_app_call_java_app.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires./*w w w  .j  a va2s.  c  om*/
 * 
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        // Client timeout
        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:timo.yarn_app_call_java_daemon.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.//ww w .j ava  2  s.  c o  m
 * 
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        // Running
        if (YarnApplicationState.RUNNING == state) {
            LOG.info("Application is running. Breaking monitoring loop.");
            return true;
        }

        // Client timeout
        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}

From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java

License:Open Source License

@Override
public PBSJobStatusLight getJobStatus(int jobId, int requestType) throws Exception {

    if (requestType > 5 || requestType < 0)
        throw new IllegalArgumentException("requestType must be [0,1,2,3,4,5]");

    char state = '*';

    if (!jobArray.containsKey(jobId))
        state = '?';

    final JobInformation ji = jobArray.get(jobId);
    final PBSJob job = ji != null ? ji.jobSpec : null;
    YarnClientService kittenClient = ji != null ? ji.kitten : null;
    ApplicationReport appReport = null;

    if (kittenClient == null || (appReport = kittenClient.getApplicationReport()) == null) {
        state = '?';
        if (jobHolds.get(jobId) != null) {
            state = 'H';
        }// w  ww  .ja v  a2  s . c  o m
    } else {
        YarnApplicationState appState = appReport.getYarnApplicationState();
        if (kittenClient.isApplicationFinished())
            state = 'E';
        else
            switch (appState) {
            case NEW:
            case NEW_SAVING:
            case ACCEPTED:
            case SUBMITTED:
                state = 'Q';
                break;
            case FAILED:
            case KILLED:
            case FINISHED:
                state = 'E';
                break;
            case RUNNING:
                state = 'R';
                break;
            default:
                state = '?';
                break;
            }
    }

    String timeUse = appReport == null ? "0"
            : Utils.makeTime(appReport.getApplicationResourceUsageReport().getVcoreSeconds());

    String appURL = appReport == null ? "" : appReport.getTrackingUrl();

    PBSJobStatusLight rtr = null;
    String nodes = null;
    List<ContainerReport> cReports = null;
    String appId = null;

    if (requestType == 0) {
        rtr = new PBSJobStatusLight(jobId, job != null ? job.getArrayTaskIds() != null : false,
                job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state,
                job != null ? job.getQueue() : null, appURL);
    } else if (requestType == 4) {
        checkOwnerOrRoot(ji);
        JobInteractiveInfo jii = ji != null ? ji.interactive : null;
        rtr = new PBSJobStatusInteractive(jobId, job != null ? job.getArrayTaskIds() != null : false,
                job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state,
                job != null ? job.getQueue() : null, appURL, jii != null ? jii.hostname : null,
                jii != null ? jii.port : -1, jii != null ? jii.secret : null);
    } else if (requestType == 5) {
        checkOwnerOrRoot(ji);
        JobDistributedInfo jid = ji != null ? ji.distributed : null;
        String secret = jid != null ? jid.secret : null;
        String[] hostnames = jid != null ? jid.hostnames.toArray(new String[0]) : null;
        int[] ports = jid != null ? jid.ports.toArray() : null;

        rtr = new PBSJobStatusDistributed(jobId, job != null ? job.getArrayTaskIds() != null : false,
                job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state,
                job != null ? job.getQueue() : null, appURL, hostnames, ports, secret);
    }
    //we need the nodes also
    else if (requestType >= 1) {
        if (kittenClient != null) {
            ApplicationId aid = kittenClient.getApplicationReport().getApplicationId();
            appId = aid.toString();
            List<ApplicationAttemptReport> aaids = yClient.getApplicationAttempts(aid);
            ApplicationAttemptId aaid = aaids.get(aaids.size() - 1).getApplicationAttemptId();
            cReports = yClient.getContainers(aaid);
            StringBuilder sNodes = new StringBuilder();
            if (cReports.size() > 0) {
                for (ContainerReport cReport : cReports) {
                    if (cReport.getContainerState() == ContainerState.RUNNING) {
                        sNodes.append(cReport.getAssignedNode().getHost());
                        sNodes.append("+");
                    }
                }
                //remove trailing ,
                sNodes.setLength(sNodes.length() - 1);
            }
            nodes = sNodes.toString();
        }
        if (requestType == 1) {
            rtr = new PBSJobStatusNodes(jobId, job.getArrayTaskIds() != null,
                    job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse,
                    state, job != null ? job.getQueue() : null, appURL, nodes);
        } else if (requestType == 2) {

            String[] tContainers;
            if (job != null) {
                tContainers = job.getArrayTaskIds() != null ? ji.array2Container.values(new String[0])
                        : new String[] { ji.taskContainerId };
            } else {
                tContainers = new String[0];
            }

            String trackingURL = appReport != null ? appReport.getTrackingUrl() : null;

            rtr = new PBSJobStatusFat(jobId, job != null ? job.getArrayTaskIds() != null : false,
                    job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse,
                    state, job != null ? job.getQueue() : null, nodes, ji != null ? ji.jobSpec : null,
                    ji != null ? ji.masterContainerId : null, tContainers, trackingURL, appId);
        } else if (requestType == 3) {
            int[] arrayIds = job != null ? JobUtils.getTaskArrayItems(job.getArrayTaskIds()) : new int[0];
            if (arrayIds == null)
                arrayIds = new int[0];
            char[] states = new char[arrayIds.length];
            //String[] walltime = new String[arrayIds.length];
            int i = 0;
            for (int arid : arrayIds) {
                String containerStatus = ji.array2Container.get(arid);
                if (containerStatus == null)
                    states[i] = 'Q';
                else if (containerStatus.equals("DONE"))
                    states[i] = 'C';
                else if (containerStatus.equals("ABORTED"))
                    states[i] = 'C';
                else {
                    states[i] = 'R';
                    //                   ContainerId c = ContainerId.fromString(containerStatus);
                    //                   for(ContainerReport cReport : cReports)
                    //                   {
                    //                     if (cReport.getContainerId().equals(c)
                    //                     {
                    //                        walltime[i] = cReport.
                    //                     }
                }
                i++;
            }

            rtr = new PBSJobArrayStatusLight(jobId, job != null ? job.getJob_Name() : null,
                    job != null ? job.getJob_Owner() : null, timeUse, state,
                    job != null ? job.getQueue() : null, appURL, arrayIds, states);
        } else { //this should not be reached.
            throw new IllegalArgumentException("Bad requestType");
        }
    }
    return rtr;
}

From source file:x10.x10rt.yarn.Client.java

License:Open Source License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires./* w ww  . ja v  a 2 s  .  c om*/
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
    YarnApplicationState previousState = null;
    while (true) {
        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);
        YarnApplicationState state = report.getYarnApplicationState();
        if (!state.equals(previousState)) {
            previousState = state;
            LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                    + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                    + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                    + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                    + ", yarnAppState=" + report.getYarnApplicationState().toString()
                    + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
                    + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
        }
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }
}

From source file:yarnkit.client.YarnClientService.java

License:Apache License

@Override
protected void runOneIteration() throws Exception {
    if (isApplicationFinished()) {
        LOG.info("Nothing to do, application is finished");
        return;//  ww w .  j a  v a 2s . c  o  m
    }

    ApplicationReport report = getApplicationReport();
    if (report == null) {
        LOG.error("No application report received");
    } else if (DONE.contains(report.getYarnApplicationState())
            || report.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
        finalReport = report;
        stop();
    }

    // Ensure that we haven't been running for all that long.
    if (parameters.getClientTimeoutMillis() > 0
            && stopwatch.elapsedMillis() > parameters.getClientTimeoutMillis()) {
        LOG.warn("Stopping application due to timeout.");
        timeout = true;
        stop();
    }
}

From source file:yarnkit.client.YarnkitClient.java

License:Apache License

private int handle(@Nonnull YarnClientService service) throws Exception {
    service.startAndWait();/*from w  w w .  j  a v  a2  s  .  c  o  m*/
    if (!service.isRunning()) {
        LOG.error("Service failed to startup");
        return 1;
    }

    String trackingUrl = null;
    while (service.isRunning()) {
        if (trackingUrl == null) {
            Thread.sleep(1000);
            ApplicationReport report = service.getApplicationReport();
            YarnApplicationState yarnAppState = report.getYarnApplicationState();
            if (yarnAppState == YarnApplicationState.RUNNING) {
                trackingUrl = report.getTrackingUrl();
                if (trackingUrl == null || trackingUrl.isEmpty()) {
                    LOG.info("Application is running, but did not specify a tracking URL");
                    trackingUrl = "";
                } else {
                    LOG.info("Master Tracking URL: " + trackingUrl);
                }
            }
        }
    }

    ApplicationReport report = service.getFinalReport();
    if (report == null) {
        LOG.error("No final report");
        return 1;
    } else if (report.getFinalApplicationStatus() != FinalApplicationStatus.SUCCEEDED) {
        LOG.error(report);
        return 1;
    } else {
        LOG.info("Final report: \n" + report);
        return 0;
    }
}

From source file:yrun.commands.Yps.java

License:Apache License

public static void main(String[] args) throws YarnException, IOException {
    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);/*from   w  w  w. j  a  v  a  2 s  . co  m*/
    yarnClient.start();
    try {
        List<ApplicationReport> applications = yarnClient.getApplications();
        for (ApplicationReport applicationReport : applications) {
            ApplicationId applicationId = applicationReport.getApplicationId();
            String user = applicationReport.getUser();
            String queue = applicationReport.getQueue();
            String name = applicationReport.getName();
            YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();
            float progress = applicationReport.getProgress();
            System.out.printf("%s\t%s\t%s\t%s\t%s\t%f%n", toString(applicationId), user, queue, name,
                    yarnApplicationState.name(), progress);
        }
    } finally {
        yarnClient.stop();
        yarnClient.close();
    }
}

From source file:yrun.YarnRunner.java

License:Apache License

public void execute() throws IOException, YarnException, InterruptedException {
    LOG.info("Using application path [" + _installPath + "]");
    Path jarPath = installThisJar(_installPath, _appJarFile);
    LOG.info("Driver installed [" + jarPath + "]");
    List<Path> installedArchivePathList = install(_installPath, _archivePathList);
    for (Path p : installedArchivePathList) {
        LOG.info("Archive installed [" + p + "]");
    }/* w ww.  j a v  a 2s.  co m*/

    YarnRunnerArgs yarnRunnerArgs = new YarnRunnerArgs();
    yarnRunnerArgs.setCommand(_command);

    Path argsPath = installThisArgs(_installPath, yarnRunnerArgs);

    final YarnClient client = YarnClient.createYarnClient();
    _configuration.setInt("yarn.nodemanager.delete.debug-delay-sec", (int) TimeUnit.HOURS.toSeconds(1));
    client.init(_configuration);
    client.start();

    YarnClientApplication app = client.createApplication();
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv, _appJarFile);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    {
        LocalResource appMasterJar = Records.newRecord(LocalResource.class);
        setupAppMasterJar(jarPath, appMasterJar);
        localResources.put(jarPath.getName(), appMasterJar);
    }
    {
        LocalResource appMasterArgs = Records.newRecord(LocalResource.class);
        setupAppMasterArgs(argsPath, appMasterArgs);
        localResources.put(MASTER_JSON, appMasterArgs);
    }

    List<String> vargs = new ArrayList<String>();
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    vargs.add("-Xmx256m");
    vargs.add("-Djava.net.preferIPv4Stack=true");
    vargs.add(YarnRunnerApplicationMaster.class.getName());

    String strCommand = "(echo ENV && set && echo CURRENT_DIR_LISTING && ls -la && echo PWD && pwd && ("
            + StringUtils.join(" ", vargs) + "))";
    strCommand += " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout";
    strCommand += " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";
    LOG.debug("Application Master command [" + strCommand + "]");

    amContainer.setCommands(Collections.singletonList(strCommand));
    amContainer.setLocalResources(localResources);
    amContainer.setEnvironment(appMasterEnv);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(_yarnName);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    if (_queue != null) {
        appContext.setQueue(_queue);
    }
    appContext.setApplicationType("yrun");

    ApplicationId appId = appContext.getApplicationId();
    AtomicBoolean shutdown = new AtomicBoolean();
    if (!_isDaemon) {
        addShutdownHook(client, appId, shutdown);
    }

    LOG.info("Submitting application with id [" + appId + "]");
    client.submitApplication(appContext);
    ApplicationReport report;
    YarnApplicationState state;
    do {
        report = client.getApplicationReport(appId);
        state = report.getYarnApplicationState();
        if (state == YarnApplicationState.RUNNING) {
            if (_isDaemon) {
                LOG.info("Application is running.  This is a daemon application driver program exiting.");
                return;
            }
        }
        Thread.sleep(100);
    } while (isNoLongerRunning(state));
    shutdown.set(true);
    LOG.info("Application has finished with state [" + state + "]");
}