Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getDiagnostics

List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getDiagnostics

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ApplicationReport
  6. getDiagnostics

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getDiagnostics.

Prototype

@Public
@Stable
public abstract String getDiagnostics();

Source Link

Document

Get the diagnositic information of the application in case of errors.

Usage

From source file:org.apache.flink.yarn.YarnClusterClient.java

License:Apache License

public ApplicationStatus getApplicationStatus() {
    if (!isConnected) {
        throw new IllegalStateException("The cluster has been connected to the ApplicationMaster.");
    }//from   w w  w.jav a2  s .com
    ApplicationReport lastReport = null;
    if (pollingRunner == null) {
        LOG.warn("YarnClusterClient.getApplicationStatus() has been called on an uninitialized cluster."
                + "The system might be in an erroneous state");
    } else {
        lastReport = pollingRunner.getLastReport();
    }
    if (lastReport == null) {
        LOG.warn(
                "YarnClusterClient.getApplicationStatus() has been called on a cluster that didn't receive a status so far."
                        + "The system might be in an erroneous state");
        return ApplicationStatus.UNKNOWN;
    } else {
        YarnApplicationState appState = lastReport.getYarnApplicationState();
        ApplicationStatus status = (appState == YarnApplicationState.FAILED
                || appState == YarnApplicationState.KILLED) ? ApplicationStatus.FAILED
                        : ApplicationStatus.SUCCEEDED;
        if (status != ApplicationStatus.SUCCEEDED) {
            LOG.warn("YARN reported application state {}", appState);
            LOG.warn("Diagnostics: {}", lastReport.getDiagnostics());
        }
        return status;
    }
}

From source file:org.apache.flink.yarn.YarnClusterClient.java

License:Apache License

/**
 * Shuts down the Yarn application/*from  w ww. j  a  v a  2  s. c o  m*/
 */
public void shutdownCluster() {

    if (hasBeenShutDown.getAndSet(true)) {
        return;
    }

    if (!isConnected) {
        throw new IllegalStateException("The cluster has been not been connected to the ApplicationMaster.");
    }

    try {
        Runtime.getRuntime().removeShutdownHook(clientShutdownHook);
    } catch (IllegalStateException e) {
        // we are already in the shutdown hook
    }

    LOG.info("Sending shutdown request to the Application Master");
    try {
        Future<Object> response = Patterns.ask(applicationClient.get(), new YarnMessages.LocalStopYarnSession(
                getApplicationStatus(), "Flink YARN Client requested shutdown"), new Timeout(akkaDuration));
        Await.ready(response, akkaDuration);
    } catch (Exception e) {
        LOG.warn("Error while stopping YARN cluster.", e);
    }

    try {
        File propertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(flinkConfig);
        if (propertiesFile.isFile()) {
            if (propertiesFile.delete()) {
                LOG.info("Deleted Yarn properties file at {}", propertiesFile.getAbsoluteFile().toString());
            } else {
                LOG.warn("Couldn't delete Yarn properties file at {}",
                        propertiesFile.getAbsoluteFile().toString());
            }
        }
    } catch (Exception e) {
        LOG.warn("Exception while deleting the JobManager address file", e);
    }

    if (sessionFilesDir != null) {
        LOG.info("Deleting files in " + sessionFilesDir);
        try {
            FileSystem shutFS = FileSystem.get(hadoopConfig);
            shutFS.delete(sessionFilesDir, true); // delete conf and jar file.
            shutFS.close();
        } catch (IOException e) {
            LOG.error("Could not delete the Flink jar and configuration files in HDFS..", e);
        }
    } else {
        LOG.warn("Session file directory not set. Not deleting session files");
    }

    try {
        pollingRunner.stopRunner();
        pollingRunner.join(1000);
    } catch (InterruptedException e) {
        LOG.warn("Shutdown of the polling runner was interrupted", e);
        Thread.currentThread().interrupt();
    }

    try {
        ApplicationReport appReport = yarnClient.getApplicationReport(appId);

        LOG.info("Application " + appId + " finished with state " + appReport.getYarnApplicationState()
                + " and final state " + appReport.getFinalApplicationStatus() + " at "
                + appReport.getFinishTime());

        if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED
                || appReport.getYarnApplicationState() == YarnApplicationState.KILLED) {
            LOG.warn("Application failed. Diagnostics " + appReport.getDiagnostics());
            LOG.warn("If log aggregation is activated in the Hadoop cluster, we recommend to retrieve "
                    + "the full application log using this command:" + System.lineSeparator()
                    + "\tyarn logs -applicationId " + appReport.getApplicationId() + System.lineSeparator()
                    + "(It sometimes takes a few seconds until the logs are aggregated)");
        }
    } catch (Exception e) {
        LOG.warn("Couldn't get final report", e);
    }

    LOG.info("YARN Client is shutting down");
    yarnClient.stop(); // actorRunner is using the yarnClient.
    yarnClient = null; // set null to clearly see if somebody wants to access it afterwards.
}

From source file:org.apache.giraph.yarn.GiraphYarnClient.java

License:Apache License

/**
 * Kill time for the client, report progress occasionally, and otherwise
 * just sleep and wait for the job to finish. If no AM response, kill the app.
 * @return true if job run is successful.
 *//*from w ww  . j  a  va2s  . c  o  m*/
private boolean awaitGiraphJobCompletion() throws YarnException, IOException {
    boolean done;
    ApplicationReport report = null;
    try {
        do {
            try {
                Thread.sleep(JOB_STATUS_INTERVAL_MSECS);
            } catch (InterruptedException ir) {
                LOG.info("Progress reporter's sleep was interrupted!", ir);
            }
            report = yarnClient.getApplicationReport(appId);
            done = checkProgress(report);
        } while (!done);
        if (!giraphConf.metricsEnabled()) {
            cleanupJarCache();
        }
    } catch (IOException ex) {
        final String diagnostics = (null == report) ? "" : "Diagnostics: " + report.getDiagnostics();
        LOG.error("Fatal fault encountered, failing " + jobName + ". " + diagnostics, ex);
        try {
            LOG.error("FORCIBLY KILLING Application from AppMaster.");
            yarnClient.killApplication(appId);
        } catch (YarnException yre) {
            LOG.error("Exception raised in attempt to kill application.", yre);
        }
        return false;
    }
    return printFinalJobReport();
}

From source file:org.apache.giraph.yarn.GiraphYarnClient.java

License:Apache License

/**
 * Assess whether job is already finished/failed and 'done' flag needs to be
 * set, prints progress display for client if all is going well.
 * @param report the application report to assess.
 * @return true if job report indicates the job run is over.
 *//*from   w w  w  . j  a  va 2 s.com*/
private boolean checkProgress(final ApplicationReport report) {
    YarnApplicationState jobState = report.getYarnApplicationState();
    if (jobState == YarnApplicationState.FINISHED || jobState == YarnApplicationState.KILLED) {
        return true;
    } else if (jobState == YarnApplicationState.FAILED) {
        LOG.error(jobName + " reports FAILED state, diagnostics show: " + report.getDiagnostics());
        return true;
    } else {
        if (reportCounter++ % 5 == 0) {
            displayJobReport(report);
        }
    }
    return false;
}

From source file:org.apache.helix.provisioning.yarn.AppLauncher.java

License:Apache License

/**
 * TODO: kill the app only in dev mode. In prod, its ok for the app to continue running if the
 * launcher dies after launching/* w ww.  j av a 2  s  .  c o m*/
 */

private String generateReport(ApplicationReport report) {
    return "Got application report from ASM for" + ", appId=" + _appId.getId() + ", clientToAMToken="
            + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost="
            + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
            + report.getYarnApplicationState().toString() + ", distributedFinalState="
            + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser();
}

From source file:org.apache.hoya.tools.HoyaUtils.java

License:Apache License

public static String appReportToString(ApplicationReport r, String separator) {
    StringBuilder builder = new StringBuilder(512);
    builder.append("application ").append(r.getName()).append("/").append(r.getApplicationType());
    builder.append(separator).append("state: ").append(r.getYarnApplicationState());
    builder.append(separator).append("URL: ").append(r.getTrackingUrl());
    builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString());
    long finishTime = r.getFinishTime();
    if (finishTime > 0) {
        builder.append(separator).append("Finished ").append(new Date(finishTime).toGMTString());
    }//from  w w  w.java2 s.  co m
    builder.append(separator).append("RPC :").append(r.getHost()).append(':').append(r.getRpcPort());
    String diagnostics = r.getDiagnostics();
    if (!diagnostics.isEmpty()) {
        builder.append(separator).append("Diagnostics :").append(diagnostics);
    }
    return builder.toString();
}

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.//from   w  w  w .ja v  a2 s . co  m
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.RUNNING == state) {
            LOG.info("Application is running...");
            return true;
        }
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:org.apache.slider.common.tools.SliderUtils.java

License:Apache License

public static String appReportToString(ApplicationReport r, String separator) {
    StringBuilder builder = new StringBuilder(512);
    builder.append("application ").append(r.getName()).append("/").append(r.getApplicationType())
            .append(separator);//w w  w.j ava2 s  .c o m
    Set<String> tags = r.getApplicationTags();
    if (!tags.isEmpty()) {
        for (String tag : tags) {
            builder.append(tag).append(separator);
        }
    }
    DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
    dateFormat.setTimeZone(TimeZone.getDefault());
    builder.append("state: ").append(r.getYarnApplicationState());
    String trackingUrl = r.getTrackingUrl();
    if (isSet(trackingUrl)) {
        builder.append(separator).append("URL: ").append(trackingUrl);
    }
    builder.append(separator).append("Started: ").append(dateFormat.format(new Date(r.getStartTime())));
    long finishTime = r.getFinishTime();
    if (finishTime > 0) {
        builder.append(separator).append("Finished: ").append(dateFormat.format(new Date(finishTime)));
    }
    String rpcHost = r.getHost();
    if (!isSet(rpcHost)) {
        builder.append(separator).append("RPC :").append(rpcHost).append(':').append(r.getRpcPort());
    }
    String diagnostics = r.getDiagnostics();
    if (!isSet(diagnostics)) {
        builder.append(separator).append("Diagnostics :").append(diagnostics);
    }
    return builder.toString();
}

From source file:org.apache.slider.core.launch.SerializedApplicationReport.java

License:Apache License

public SerializedApplicationReport(ApplicationReport report) {
    this.applicationId = report.getApplicationId().toString();
    this.applicationAttemptId = report.getCurrentApplicationAttemptId().toString();
    this.name = report.getName();
    this.applicationType = report.getApplicationType();
    this.user = report.getUser();
    this.queue = report.getQueue();
    this.host = report.getHost();
    this.rpcPort = report.getRpcPort();
    this.state = report.getYarnApplicationState().toString();
    this.diagnostics = report.getDiagnostics();
    this.startTime = report.getStartTime();
    this.finishTime = report.getFinishTime();
    this.finalStatus = report.getFinalApplicationStatus().toString();
    this.progress = report.getProgress();
}

From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java

License:Apache License

private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws IOException, YarnException {

    long sleepTime = 100;
    int count = 1;
    while (true) {
        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }//from  w  w w.  jav a 2s .co  m
        try {
            Thread.sleep(sleepTime);
            sleepTime = count * 100;
            if (count < 10) {
                count++;
            }
        } catch (InterruptedException e) {
            //LOG.debug("Thread sleep in monitoring loop interrupted");
        }
    }
}