Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getStartTime

List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getStartTime

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ApplicationReport
  6. getStartTime

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getStartTime.

Prototype

@Public
@Stable
public abstract long getStartTime();

Source Link

Document

Get the start time of the application.

Usage

From source file:com.sogou.dockeronyarn.client.DockerClient.java

License:Apache License

/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. //ww  w .  j  a va  2 s  .  c o  m
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }
        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not shutdown." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:com.twitter.hraven.hadoopJobMonitor.policy.DefaultPolicy.java

License:Apache License

/**
 * Check the status of an application/*  w  w w . j  a v  a 2  s  .c o m*/
 * 
 * @param appReport
 * @param appConf
 * @return true if the app is well-behaved
 */
@Override
public String checkAppStatus(ApplicationReport appReport, AppConfiguraiton appConf) {
    final long start = appReport.getStartTime();
    final long currTime = System.currentTimeMillis();
    final long duration = currTime - start;
    final long maxJobLenMs = appConf.getMaxJobLenMin() * 60 * 1000;
    if (duration > maxJobLenMs) {
        String errMsg = Notifier.tooLongApp(appConf, appReport, duration, maxJobLenMs);
        if (appConf.isEnforced(HadoopJobMonitorConfiguration.JOB_MAX_LEN_MIN))
            return errMsg;
    }
    return null;
}

From source file:com.zqh.hadoop.moya.core.yarn.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires.//from  w ww  .  ja v  a 2  s  . c  o  m
 *
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws org.apache.hadoop.yarn.exceptions.YarnException
 * @throws java.io.IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }

}

From source file:io.hops.hopsworks.api.zeppelin.rest.InterpreterRestApi.java

License:Apache License

private List<JobAdministration.YarnApplicationReport> fetchJobs() {
    JobAdministration jobAdmin = new JobAdministration();
    List<JobAdministration.YarnApplicationReport> reports = new ArrayList<>();
    YarnClient client = YarnClient.createYarnClient();
    Configuration conf = settings.getConfiguration();
    client.init(conf);//  w w  w .  j av a  2 s  .  c o  m
    client.start();
    try {
        //Create our custom YarnApplicationReport Pojo
        for (ApplicationReport appReport : client.getApplications(PREDICATE)) {
            reports.add(jobAdmin.new YarnApplicationReport(appReport.getApplicationId().toString(),
                    appReport.getName(), appReport.getUser(), appReport.getStartTime(),
                    appReport.getFinishTime(), appReport.getApplicationId().getClusterTimestamp(),
                    appReport.getApplicationId().getId(), appReport.getYarnApplicationState().name()));
        }
    } catch (YarnException | IOException ex) {
        logger.error("", ex);
    }
    return reports;
}

From source file:io.hops.hopsworks.api.zeppelin.rest.InterpreterRestApi.java

License:Apache License

private List<JobAdministration.YarnApplicationReport> fetchJobs(String username) {
    JobAdministration jobAdmin = new JobAdministration();
    List<JobAdministration.YarnApplicationReport> reports = new ArrayList<>();
    YarnClient client = YarnClient.createYarnClient();
    Configuration conf = settings.getConfiguration();
    client.init(conf);/*from www  . j  a  va2  s.com*/
    client.start();
    try {
        //Create our custom YarnApplicationReport Pojo
        for (ApplicationReport appReport : client.getApplications(PREDICATE)) {
            if (username.equals(appReport.getUser())) {
                reports.add(jobAdmin.new YarnApplicationReport(appReport.getApplicationId().toString(),
                        appReport.getName(), appReport.getUser(), appReport.getStartTime(),
                        appReport.getFinishTime(), appReport.getApplicationId().getClusterTimestamp(),
                        appReport.getApplicationId().getId(), appReport.getYarnApplicationState().name()));
            }
        }
    } catch (YarnException | IOException ex) {
        logger.error("", ex);
    }
    return reports;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires./*  w ww . j  a v a2 s.  co m*/
 *
 * @param appId
 *     Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
public boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    YarnApplicationState oldState = null;

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);
        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();

        if (oldState != state) {
            LOG.info("Got application report from ASM for" + "\n\t appId=" + appId.getId()
                    + "\n\t clientToAMToken=" + report.getClientToAMToken() + "\n\t appDiagnostics="
                    + report.getDiagnostics() + "\n\t appMasterHost=" + report.getHost() + "\n\t appQueue="
                    + report.getQueue() + "\n\t appMasterRpcPort=" + report.getRpcPort() + "\n\t appStartTime="
                    + report.getStartTime() + "\n\t yarnAppState=" + report.getYarnApplicationState().toString()
                    + "\n\t distributedFinalState=" + report.getFinalApplicationStatus().toString()
                    + "\n\t appTrackingUrl=" + report.getTrackingUrl() + "\n\t appUser=" + report.getUser());
            oldState = state;
        } else {
            LOG.info("Got application report from ASM for " + appId + " (state: " + state + ")");
        }

        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}

From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java

License:Apache License

/**
 * Assess whether job is already finished/failed and 'done' flag needs to be
 * set, prints progress display for client if all is going well.
 * //from   w ww . j a  v  a2  s . co  m
 * @param report
 *            the application report to assess.
 * @return true if job report indicates the job run is over.
 */
private boolean checkProgress(final ApplicationReport report) {
    YarnApplicationState jobState = report.getYarnApplicationState();

    LOG.info(
            "Got applicaton report for appId={}, state={}, progress={}%, amDiag={}, masterHost={}, masterRpcPort={}, queue={}, startTime={}, clientToken={}, finalState={}, trackingUrl={}, user={}",
            this.appId.getId(), report.getYarnApplicationState().toString(),
            DF.format(report.getProgress() * 100), report.getDiagnostics(), report.getHost(),
            report.getRpcPort(), report.getQueue(), report.getStartTime(), report.getClientToAMToken(),
            report.getFinalApplicationStatus().toString(), report.getTrackingUrl(), report.getUser());
    switch (jobState) {
    case FINISHED:
        LOG.info("Application finished in {} ms", (System.currentTimeMillis() - getStartTime()));
        return true;
    case KILLED:
        LOG.error("{} reports KILLED state, diagnostics show: {}", getAppName(), report.getDiagnostics());
        return true;
    case FAILED:
        LOG.error("{} reports FAILED state, diagnostics show: {}", getAppName(), report.getDiagnostics());
        return true;
    default:
        if (this.reportCounter++ % 5 == 0) {
            displayJobReport(report);
        }
        return false;
    }
}

From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java

License:Apache License

/**
 * Display a formatted summary of the job progress report from the AM.
 * //from w w  w. ja va2  s.  c  om
 * @param report
 *            the report to display.
 */
private void displayJobReport(final ApplicationReport report) {
    if (null == report) {
        throw new IllegalStateException(String.format(
                "[*] Latest ApplicationReport for job %s was not received by the local client.", getAppName()));
    }
    final float elapsed = (System.currentTimeMillis() - report.getStartTime()) / 1000.0f;
    LOG.info("{}, Elapsed: {}", getAppName(), String.format("%.2f secs", elapsed));
    LOG.info("{}, State: {} , Containers: used/reserved/needed-resources {}/{}/{}",
            report.getCurrentApplicationAttemptId(), report.getYarnApplicationState().name(),
            report.getApplicationResourceUsageReport().getNumUsedContainers(),
            report.getApplicationResourceUsageReport().getNumReservedContainers(),
            report.getApplicationResourceUsageReport().getNeededResources());
}

From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java

License:Apache License

private int printFinalJobReport() throws YarnException, IOException {
    try {/*from   www  .  j  av  a2 s .  c o  m*/
        ApplicationReport report = this.yarnClient.getApplicationReport(getAppId());
        FinalApplicationStatus finalAppStatus = report.getFinalApplicationStatus();
        final long secs = (report.getFinishTime() - report.getStartTime()) / 1000L;
        final String time = String.format("%d minutes, %d seconds.", secs / 60L, secs % 60L);
        LOG.info("Completed {}: {}, total running time: {}", getAppName(), finalAppStatus.name(), time);
        return finalAppStatus == FinalApplicationStatus.SUCCEEDED ? 0 : -1;
    } catch (YarnException yre) {
        LOG.error(String.format("Exception encountered while attempting to request a final job report for %s.",
                getAppId()), yre);
        return -1;
    }
}

From source file:org.apache.ambari.view.slider.SliderAppsViewControllerImpl.java

License:Apache License

private SliderApp createSliderAppObject(ApplicationReport yarnApp, Set<String> properties,
        SliderClient sliderClient) {//from   www.  j a v a 2s .c o m
    if (yarnApp == null) {
        return null;
    }

    SliderApp app = new SliderApp();
    app.setState(yarnApp.getYarnApplicationState().name());

    // Valid Slider App?
    // We want all Slider apps except the ones which properly finished.
    if (YarnApplicationState.FINISHED.equals(yarnApp.getYarnApplicationState())) {
        try {
            if (sliderClient.actionExists(yarnApp.getName(), false) == LauncherExitCodes.EXIT_SUCCESS) {
                app.setState(SliderApp.STATE_FROZEN);
            }
        } catch (UnknownApplicationInstanceException e) {
            return null; // Application not in HDFS - means it is not frozen
        } catch (YarnException e) {
            logger.warn("Unable to determine frozen state for " + yarnApp.getName(), e);
            return null;
        } catch (IOException e) {
            logger.warn("Unable to determine frozen state for " + yarnApp.getName(), e);
            return null;
        }
    }
    if (YarnApplicationState.KILLED.equals(yarnApp.getYarnApplicationState())
            || YarnApplicationState.FAILED.equals(yarnApp.getYarnApplicationState())) {
        try {
            if (sliderClient.actionExists(yarnApp.getName(), false) != LauncherExitCodes.EXIT_SUCCESS) {
                // YARN application is killed or failed, and no HDFS content - Application has been destroyed.
                return null;
            }
        } catch (UnknownApplicationInstanceException e) {
            return null; // Application not in HDFS - means it is not frozen
        } catch (YarnException e) {
            logger.warn("Unable to determine status of killed app " + yarnApp.getName(), e);
            return null;
        } catch (IOException e) {
            logger.warn("Unable to determine status of killed app " + yarnApp.getName(), e);
            return null;
        }
    }

    app.setId(getApplicationIdString(yarnApp.getApplicationId()));
    app.setName(yarnApp.getName());
    app.setUser(yarnApp.getUser());
    app.setDiagnostics(yarnApp.getDiagnostics());
    app.setYarnId(yarnApp.getApplicationId().toString());
    app.setStartTime(yarnApp.getStartTime());
    app.setEndTime(yarnApp.getFinishTime());
    Set<String> applicationTags = yarnApp.getApplicationTags();
    if (applicationTags != null && applicationTags.size() > 0) {
        for (String tag : applicationTags) {
            int index = tag.indexOf(':');
            if (index > 0 && index < tag.length() - 1) {
                String key = tag.substring(0, index).trim();
                String value = tag.substring(index + 1).trim();
                if ("name".equals(key)) {
                    app.setType(value);
                } else if ("version".equals(key)) {
                    app.setAppVersion(value);
                } else if ("description".equals(key)) {
                    app.setDescription(value);
                }
            }
        }
    }
    if (properties != null && !properties.isEmpty()) {
        SliderAppType matchedAppType = null;
        List<SliderAppType> matchingAppTypes = getSliderAppTypes(null);
        if (matchingAppTypes != null && matchingAppTypes.size() > 0) {
            for (SliderAppType appType : matchingAppTypes) {
                if ((appType.getTypeName() != null && appType.getTypeName().equalsIgnoreCase(app.getType()))
                        && (appType.getTypeVersion() != null
                                && appType.getTypeVersion().equalsIgnoreCase(app.getAppVersion()))) {
                    matchedAppType = appType;
                    break;
                }
            }
        }

        SliderAppMasterClient sliderAppClient = yarnApp.getTrackingUrl() == null ? null
                : new SliderAppMasterClient(yarnApp.getTrackingUrl());
        SliderAppMasterData appMasterData = null;
        Map<String, String> quickLinks = new HashMap<String, String>();
        Set<String> metrics = new HashSet<String>();
        for (String property : properties) {
            if ("RUNNING".equals(app.getState())) {
                if (sliderAppClient != null) {
                    if (appMasterData == null) {
                        appMasterData = sliderAppClient.getAppMasterData();
                    }
                    if (appMasterData != null && "urls".equals(property.toLowerCase())) {
                        if (quickLinks.isEmpty()) {
                            quickLinks = sliderAppClient.getQuickLinks(appMasterData.publisherUrl);
                        }
                        app.setUrls(quickLinks);
                    } else if (appMasterData != null && "configs".equals(property.toLowerCase())) {
                        Map<String, Map<String, String>> configs = sliderAppClient
                                .getConfigs(appMasterData.publisherUrl);
                        app.setConfigs(configs);
                    } else if (appMasterData != null && "jmx".equals(property.toLowerCase())) {
                        if (quickLinks.isEmpty()) {
                            quickLinks = sliderAppClient.getQuickLinks(appMasterData.publisherUrl);
                        }
                        if (quickLinks != null && quickLinks.containsKey("JMX")) {
                            String jmxUrl = quickLinks.get("JMX");
                            if (matchedAppType != null) {
                                MetricsHolder metricsHolder = appMetrics.get(matchedAppType.uniqueName());
                                app.setJmx(sliderAppClient.getJmx(jmxUrl, viewContext, matchedAppType,
                                        metricsHolder));
                            }
                        }
                        Map<String, Map<String, String>> configs = sliderAppClient
                                .getConfigs(appMasterData.publisherUrl);
                        app.setConfigs(configs);
                    } else if ("components".equals(property.toLowerCase())) {
                        try {
                            ClusterDescription description = sliderClient
                                    .getClusterDescription(yarnApp.getName());
                            if (description != null && description.status != null
                                    && !description.status.isEmpty()) {
                                Map<String, SliderAppComponent> componentTypeMap = new HashMap<String, SliderAppComponent>();
                                for (Entry<String, Object> e : description.status.entrySet()) {
                                    @SuppressWarnings("unchecked")
                                    Map<String, Map<String, Map<String, Object>>> componentsObj = (Map<String, Map<String, Map<String, Object>>>) e
                                            .getValue();
                                    boolean isLive = "live".equals(e.getKey());
                                    for (Entry<String, Map<String, Map<String, Object>>> componentEntry : componentsObj
                                            .entrySet()) {
                                        SliderAppComponent appComponent = componentTypeMap
                                                .get(componentEntry.getKey());
                                        if (appComponent == null) {
                                            appComponent = new SliderAppComponent();
                                            appComponent.setComponentName(componentEntry.getKey());
                                            appComponent.setActiveContainers(
                                                    new HashMap<String, Map<String, String>>());
                                            appComponent.setCompletedContainers(
                                                    new HashMap<String, Map<String, String>>());
                                            componentTypeMap.put(componentEntry.getKey(), appComponent);
                                        }
                                        for (Entry<String, Map<String, Object>> containerEntry : componentEntry
                                                .getValue().entrySet()) {
                                            Map<String, String> containerDataMap = new HashMap<String, String>();
                                            String containerId = containerEntry.getKey();
                                            Map<String, Object> containerValues = containerEntry.getValue();
                                            for (String containerProperty : containerValues.keySet()) {
                                                Object containerPropertyValue = containerValues
                                                        .get(containerProperty);
                                                containerDataMap.put(containerProperty,
                                                        containerPropertyValue.toString());
                                            }
                                            if (isLive) {
                                                appComponent.getActiveContainers().put(containerId,
                                                        containerDataMap);
                                            } else {
                                                appComponent.getCompletedContainers().put(containerId,
                                                        containerDataMap);
                                            }
                                        }
                                        // Set total instances count from statistics
                                        appComponent.setInstanceCount(appComponent.getActiveContainers().size()
                                                + appComponent.getCompletedContainers().size());
                                        if (description.statistics != null && description.statistics
                                                .containsKey(componentEntry.getKey())) {
                                            Map<String, Integer> statisticsMap = description.statistics
                                                    .get(componentEntry.getKey());
                                            if (statisticsMap.containsKey("containers.desired")) {
                                                appComponent.setInstanceCount(
                                                        statisticsMap.get("containers.desired"));
                                            }
                                        }
                                    }
                                }
                                app.setAlerts(
                                        sliderAlerts.generateComponentsAlerts(componentTypeMap, app.getType()));
                                app.setComponents(componentTypeMap);
                            }
                        } catch (UnknownApplicationInstanceException e) {
                            logger.warn("Unable to determine app components for " + yarnApp.getName(), e);
                        } catch (YarnException e) {
                            logger.warn("Unable to determine app components for " + yarnApp.getName(), e);
                            throw new RuntimeException(e.getMessage(), e);
                        } catch (IOException e) {
                            logger.warn("Unable to determine app components for " + yarnApp.getName(), e);
                            throw new RuntimeException(e.getMessage(), e);
                        }
                    } else if (property.startsWith(METRICS_PREFIX)) {
                        metrics.add(property.substring(METRICS_PREFIX.length()));
                    } else if ("supportedMetrics".equals(property)) {
                        if (matchedAppType != null) {
                            app.setSupportedMetrics(matchedAppType.getSupportedMetrics());
                        }
                    }
                }
            }
        }
        if (metrics.size() > 0) {
            if (quickLinks.isEmpty()) {
                quickLinks = sliderAppClient.getQuickLinks(appMasterData.publisherUrl);
            }
            if (quickLinks != null && quickLinks.containsKey(METRICS_API_NAME)) {
                String metricsUrl = quickLinks.get(METRICS_API_NAME);
                MetricsHolder metricsHolder = appMetrics.get(matchedAppType.uniqueName());
                app.setMetrics(sliderAppClient.getMetrics(yarnApp.getName(), metricsUrl, metrics, null,
                        viewContext, matchedAppType, metricsHolder));
            }
        }
    }
    return app;
}