Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getApplicationId

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getApplicationId.

Prototype

@Public
@Stable
public abstract ApplicationId getApplicationId();

Source Link

Document

Get the ApplicationId of the application.

Usage

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

private boolean isInWhitelist(ApplicationReport appReport) {
    String appId = appReport.getApplicationId().toString();
    if (vConf.isUserInWhitelist(appReport.getUser()))
        return true;
    if (WhiteList.getInstance().isWhiteListed(appId))
        return true;
    return false;
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

private AppConfiguraiton getAppConf(ApplicationReport appReport) throws ConfigurationAccessException {
    ApplicationId appId = appReport.getApplicationId();
    AppConfiguraiton appConf = appConfCache.get(appId);
    if (appConf != null)
        return appConf;
    String xmlUrlStr = buildXmlUrl(appReport);
    URL xmlUrl;/*www . j a v  a2  s.  c o m*/
    try {
        xmlUrl = new URL(xmlUrlStr);
        Configuration origAppConf = new Configuration(false);
        origAppConf.addResource(xmlUrl);
        appConf = new AppConfiguraiton(origAppConf, vConf);
        appConfCache.put(appId, appConf);
    } catch (MalformedURLException e) {
        LOG.warn(e);
        throw new AppConfiguraiton.ConfigurationAccessException(e);
    }
    return appConf;
}

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

/**
 * kill an application./*from   w w w .  j a v a  2  s  . c o  m*/
 * 
 * @param appReport
 */
private void killApp(ApplicationReport appReport, String errMsg) {
    LOG.info("App kill request " + appReport.getApplicationId());
    if (vConf.isDryRun())
        return;
    LOG.warn("KILLING job " + appReport.getApplicationId());
    try {
        metrics.killedApps.incr();
        //leave it for when the new API make it to a release of hadoop
        //      rmDelegate.killApplication(appReport.getApplicationId(), "killed by hadoopJobMonitor: "+errMsg);
        rmDelegate.killApplication(appReport.getApplicationId());
    } catch (YarnRuntimeException e) {
        LOG.error("Error in killing job " + appReport.getApplicationId(), e);
    } catch (YarnException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

From source file:com.twitter.hraven.hadoopJobMonitor.ClusterStatusChecker.java

License:Apache License

/**
 * The pre-check performed on the app status. The idea is to reduce the avoid
 * paying the cost of the actual check, if it app does not need it.
 * /*from  w  w  w .  j a  va 2s .co m*/
 * @param appReport
 * @return true if app passes this pre-check
 */
private boolean preAppStatusCheck(ApplicationReport appReport) {
    YarnApplicationState appState = appReport.getYarnApplicationState();
    LOG.debug("checking app " + appReport.getApplicationId() + " state is " + appState);
    return appState == YarnApplicationState.RUNNING;
}

From source file:com.twitter.hraven.hadoopJobMonitor.ClusterStatusChecker.java

License:Apache License

/**
 * Check the status of an application.//from  www.  j a v a 2  s . c  om
 * 
 * @param appReport
 */
private void checkAppStatus(ApplicationReport appReport) {
    final ApplicationId appId = appReport.getApplicationId();
    if (isInWhitelist(appReport)) {
        LOG.warn("Skip whitelisted app " + appId + " from user " + appReport.getUser());
        return;
    }
    if (runningAppCheckers.contains(appReport.getApplicationId())) {
        LOG.warn("Skip already-being-checked app " + appId + " from user " + appReport.getUser());
        return;
    } else {
        runningAppCheckers.add(appReport.getApplicationId());
    }
    boolean passPreCheck = preAppStatusCheck(appReport);
    if (!passPreCheck)
        return;
    // 2. spawn a new thread to check on each app
    LOG.debug("Spawning a thread to check on app " + appReport.getApplicationId());
    // if (!appReport.getUser().equals("myabandeh"))
    // return;
    AppCheckerProgress appCheckerProgress = new AppCheckerProgress() {
        @Override
        public void finished() {
            runningAppCheckers.remove(appId);
        }
    };
    appCheckerExecutor
            .execute(new AppStatusChecker(vConf, appReport, clientCache, rmDelegate, appCheckerProgress));
}

From source file:com.twitter.hraven.hadoopJobMonitor.notification.Notifier.java

License:Apache License

public static String longAppMsg(ApplicationReport appReport, long duration, long max) {
    String msg = "The app " + appReport.getApplicationId() + " of  " + appReport.getName() + " run by "
            + appReport.getUser() + " on " + CLUSTER + " did not finish after " + durationStr(duration)
            + " hours (Max is " + durationStr(max) + ")\n";
    msg += "Tracking url is " + appReport.getTrackingUrl() + "\n";
    return msg;//from  w w w.j a  v  a2s . c o  m
}

From source file:com.twitter.hraven.hadoopJobMonitor.policy.AppPassPolicy.java

License:Apache License

@Override
public String checkAppStatus(ApplicationReport appReport, AppConfiguraiton appConf) {
    LOG.info("PASS " + appReport.getApplicationId());
    return null;//from   ww w.j av  a 2 s . c o  m
}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

@VisibleForTesting
Optional<ApplicationId> getReconnectableApplicationId() throws YarnException, IOException {
    List<ApplicationReport> applicationReports = this.yarnClient.getApplications(APPLICATION_TYPES,
            RECONNECTABLE_APPLICATION_STATES);
    if (applicationReports == null || applicationReports.isEmpty()) {
        return Optional.absent();
    }//w w w .j  av  a 2 s .com

    // Try to find an application with a matching application name
    for (ApplicationReport applicationReport : applicationReports) {
        if (this.applicationName.equals(applicationReport.getName())) {
            return Optional.of(applicationReport.getApplicationId());
        }
    }

    return Optional.absent();
}

From source file:io.hops.hopsworks.api.zeppelin.rest.InterpreterRestApi.java

License:Apache License

private List<JobAdministration.YarnApplicationReport> fetchJobs() {
    JobAdministration jobAdmin = new JobAdministration();
    List<JobAdministration.YarnApplicationReport> reports = new ArrayList<>();
    YarnClient client = YarnClient.createYarnClient();
    Configuration conf = settings.getConfiguration();
    client.init(conf);// w w w .ja va 2  s.c o  m
    client.start();
    try {
        //Create our custom YarnApplicationReport Pojo
        for (ApplicationReport appReport : client.getApplications(PREDICATE)) {
            reports.add(jobAdmin.new YarnApplicationReport(appReport.getApplicationId().toString(),
                    appReport.getName(), appReport.getUser(), appReport.getStartTime(),
                    appReport.getFinishTime(), appReport.getApplicationId().getClusterTimestamp(),
                    appReport.getApplicationId().getId(), appReport.getYarnApplicationState().name()));
        }
    } catch (YarnException | IOException ex) {
        logger.error("", ex);
    }
    return reports;
}

From source file:io.hops.hopsworks.api.zeppelin.rest.InterpreterRestApi.java

License:Apache License

private List<JobAdministration.YarnApplicationReport> fetchJobs(String username) {
    JobAdministration jobAdmin = new JobAdministration();
    List<JobAdministration.YarnApplicationReport> reports = new ArrayList<>();
    YarnClient client = YarnClient.createYarnClient();
    Configuration conf = settings.getConfiguration();
    client.init(conf);/* w ww .j a v a 2 s . co  m*/
    client.start();
    try {
        //Create our custom YarnApplicationReport Pojo
        for (ApplicationReport appReport : client.getApplications(PREDICATE)) {
            if (username.equals(appReport.getUser())) {
                reports.add(jobAdmin.new YarnApplicationReport(appReport.getApplicationId().toString(),
                        appReport.getName(), appReport.getUser(), appReport.getStartTime(),
                        appReport.getFinishTime(), appReport.getApplicationId().getClusterTimestamp(),
                        appReport.getApplicationId().getId(), appReport.getYarnApplicationState().name()));
            }
        }
    } catch (YarnException | IOException ex) {
        logger.error("", ex);
    }
    return reports;
}