Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getDiagnostics

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getDiagnostics.

Prototype

@Public
@Stable
public abstract String getDiagnostics();

Source Link

Document

Get the diagnositic information of the application in case of errors.

Usage

From source file:org.apache.tez.client.TezClient.java

License:Apache License

/**
 * Get the status of the App Master executing the DAG
 * In non-session mode it returns the status of the last submitted DAG App Master 
 * In session mode, it returns the status of the App Master hosting the session
 * /*from  w  ww .  j  av a 2 s.  com*/
 * @return State of the session
 * @throws TezException
 * @throws IOException
 */
public synchronized TezAppMasterStatus getAppMasterStatus() throws TezException, IOException {
    // Supporting per-DAG app master case since user may choose to run the same 
    // code in that mode and the code should continue to work. Its easy to provide 
    // the correct view for per-DAG app master too.
    ApplicationId appId = null;
    if (isSession) {
        appId = sessionAppId;
    } else {
        appId = lastSubmittedAppId;
    }
    Preconditions.checkState(appId != null, "Cannot get status without starting an application");
    try {
        ApplicationReport appReport = frameworkClient.getApplicationReport(appId);
        switch (appReport.getYarnApplicationState()) {
        case NEW:
        case NEW_SAVING:
        case ACCEPTED:
        case SUBMITTED:
            return TezAppMasterStatus.INITIALIZING;
        case FAILED:
        case KILLED:
            diagnostics = appReport.getDiagnostics();
            LOG.info("App did not succeed. Diagnostics: "
                    + (appReport.getDiagnostics() != null ? appReport.getDiagnostics()
                            : NO_CLUSTER_DIAGNOSTICS_MSG));
            return TezAppMasterStatus.SHUTDOWN;
        case FINISHED:
            return TezAppMasterStatus.SHUTDOWN;
        case RUNNING:
            if (!isSession) {
                return TezAppMasterStatus.RUNNING;
            }
            try {
                DAGClientAMProtocolBlockingPB proxy = getSessionAMProxy(appId);
                if (proxy == null) {
                    return TezAppMasterStatus.INITIALIZING;
                }
                GetAMStatusResponseProto response = proxy.getAMStatus(null,
                        GetAMStatusRequestProto.newBuilder().build());
                return DagTypeConverters.convertTezSessionStatusFromProto(response.getStatus());
            } catch (TezException e) {
                LOG.info("Failed to retrieve AM Status via proxy", e);
            } catch (ServiceException e) {
                LOG.info("Failed to retrieve AM Status via proxy", e);
            }
        }
    } catch (YarnException e) {
        throw new TezException(e);
    }
    return TezAppMasterStatus.INITIALIZING;
}

---

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

static DAGClientAMProtocolBlockingPB getSessionAMProxy(FrameworkClient yarnClient, Configuration conf,
        ApplicationId applicationId) throws TezException, IOException {
    ApplicationReport appReport;
    try {// w  ww .j  a  v a  2s  . c  om
        appReport = yarnClient.getApplicationReport(applicationId);

        if (appReport == null) {
            throw new TezUncheckedException(
                    "Could not retrieve application report" + " from YARN, applicationId=" + applicationId);
        }
        YarnApplicationState appState = appReport.getYarnApplicationState();
        if (appState != YarnApplicationState.RUNNING) {
            if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED
                    || appState == YarnApplicationState.FAILED) {
                String msg = "Application not running" + ", applicationId=" + applicationId
                        + ", yarnApplicationState=" + appReport.getYarnApplicationState()
                        + ", finalApplicationStatus=" + appReport.getFinalApplicationStatus() + ", trackingUrl="
                        + appReport.getTrackingUrl() + ", diagnostics="
                        + (appReport.getDiagnostics() != null ? appReport.getDiagnostics()
                                : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG);
                LOG.info(msg);
                throw new SessionNotRunning(msg);
            }
            return null;
        }
    } catch (YarnException e) {
        throw new TezException(e);
    }
    return getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(), appReport.getClientToAMToken());
}

---

From source file:org.apache.tez.client.TezYarnClient.java

License:Apache License

@Override
public ApplicationId submitApplication(ApplicationSubmissionContext appSubmissionContext)
        throws YarnException, IOException {
    ApplicationId appId = yarnClient.submitApplication(appSubmissionContext);
    ApplicationReport appReport = getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
        throw new IOException("Failed to submit application to YARN: " + appReport.getDiagnostics());
    }// www. j  a  v  a 2  s.co m
    return appId;
}

---

From source file:org.apache.tez.dag.api.client.DAGClientImpl.java

License:Apache License

DAGStatus getDAGStatusViaRM() throws TezException, IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("GetDAGStatus via AM for app: " + appId + " dag:" + dagId);
    }/*from  w ww.  ja  va  2 s  .c  om*/
    ApplicationReport appReport;
    try {
        appReport = frameworkClient.getApplicationReport(appId);
    } catch (YarnException e) {
        throw new TezException(e);
    }

    if (appReport == null) {
        throw new TezException("Unknown/Invalid appId: " + appId);
    }

    DAGProtos.DAGStatusProto.Builder builder = DAGProtos.DAGStatusProto.newBuilder();
    DAGStatus dagStatus = new DAGStatus(builder);
    DAGProtos.DAGStatusStateProto dagState;
    switch (appReport.getYarnApplicationState()) {
    case NEW:
    case NEW_SAVING:
    case SUBMITTED:
    case ACCEPTED:
        dagState = DAGProtos.DAGStatusStateProto.DAG_SUBMITTED;
        break;
    case RUNNING:
        dagState = DAGProtos.DAGStatusStateProto.DAG_RUNNING;
        break;
    case FAILED:
        dagState = DAGProtos.DAGStatusStateProto.DAG_FAILED;
        break;
    case KILLED:
        dagState = DAGProtos.DAGStatusStateProto.DAG_KILLED;
        break;
    case FINISHED:
        switch (appReport.getFinalApplicationStatus()) {
        case UNDEFINED:
        case FAILED:
            dagState = DAGProtos.DAGStatusStateProto.DAG_FAILED;
            break;
        case KILLED:
            dagState = DAGProtos.DAGStatusStateProto.DAG_KILLED;
            break;
        case SUCCEEDED:
            dagState = DAGProtos.DAGStatusStateProto.DAG_SUCCEEDED;
            break;
        default:
            throw new TezUncheckedException("Encountered unknown final application" + " status from YARN"
                    + ", appState=" + appReport.getYarnApplicationState() + ", finalStatus="
                    + appReport.getFinalApplicationStatus());
        }
        break;
    default:
        throw new TezUncheckedException("Encountered unknown application state" + " from YARN, appState="
                + appReport.getYarnApplicationState());
    }

    builder.setState(dagState);
    if (appReport.getDiagnostics() != null) {
        builder.addAllDiagnostics(Collections.singleton(appReport.getDiagnostics()));
    }

    return dagStatus;
}

---

From source file:org.apache.tez.test.TestExceptionPropagation.java

License:Apache License

/**
 * verify the diagnostics in {@link DAGStatus} is correct in non-session mode,
 * and also verify that diagnostics from {@link DAGStatus} should match that
 * from {@link ApplicationReport}/*  w  w w  . j  ava2  s .c om*/
 * 
 * @throws Exception
 */
@Test(timeout = 120000)
public void testExceptionPropagationNonSession() throws Exception {
    try {
        startMiniTezCluster();
        startNonSessionClient();

        ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs;
        LOG.info("NonSession mode, Test for Exception from:" + exLocation.name());
        DAG dag = createDAG(exLocation);
        DAGClient dagClient = tezClient.submitDAG(dag);
        DAGStatus dagStatus = dagClient.waitForCompletion();
        String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
        LOG.info("Diagnostics:" + diagnostics);
        assertTrue(diagnostics.contains(exLocation.name()));

        // wait for app complete (unregisterApplicationMaster is done)
        ApplicationId appId = tezClient.getAppMasterApplicationId();
        YarnClient yarnClient = YarnClient.createYarnClient();
        yarnClient.init(tezConf);
        yarnClient.start();
        Set<YarnApplicationState> FINAL_APPLICATION_STATES = EnumSet.of(YarnApplicationState.KILLED,
                YarnApplicationState.FAILED, YarnApplicationState.FINISHED);
        ApplicationReport appReport = null;
        while (true) {
            appReport = yarnClient.getApplicationReport(appId);
            Thread.sleep(1000);
            LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
            LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
            if (FINAL_APPLICATION_STATES.contains(appReport.getYarnApplicationState())) {
                break;
            }
        }
        // wait for 1 second and call getApplicationReport again to ensure get the
        // diagnostics
        // TODO remove it after YARN-2560
        Thread.sleep(1000);
        appReport = yarnClient.getApplicationReport(appId);

        LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
        LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
        assertTrue(appReport.getDiagnostics().contains(exLocation.name()));
        // use "\n" as separator, because we also use it in Tez internally when
        // assembling the application diagnostics.
        assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(),
                appReport.getDiagnostics().trim());
    } finally {
        stopNonSessionClient();
        Thread.sleep(10 * 1000);
        stopTezMiniCluster();
    }
}

---

From source file:org.deeplearning4j.iterativereduce.runtime.yarn.client.Client.java

License:Apache License

/**
 * TODO: consider the scenarios where we dont get enough containers 
 * - we need to re-submit the job till we get the containers alloc'd
 * //from w w w  .  j a v a 2 s .  co m
 */
@Override
public int run(String[] args) throws Exception {

    //System.out.println("IR: Client.run() [start]");

    if (args.length < 1)
        LOG.info("No configuration file specified, using default (" + ConfigFields.DEFAULT_CONFIG_FILE + ")");

    long startTime = System.currentTimeMillis();
    String configFile = (args.length < 1) ? ConfigFields.DEFAULT_CONFIG_FILE : args[0];
    Properties props = new Properties();
    Configuration conf = getConf();

    try {
        FileInputStream fis = new FileInputStream(configFile);
        props.load(fis);
    } catch (FileNotFoundException ex) {
        throw ex; // TODO: be nice
    } catch (IOException ex) {
        throw ex; // TODO: be nice
    }

    // Make sure we have some bare minimums
    ConfigFields.validateConfig(props);

    if (LOG.isDebugEnabled()) {
        LOG.debug("Loaded configuration: ");
        for (Map.Entry<Object, Object> entry : props.entrySet()) {
            LOG.debug(entry.getKey() + "=" + entry.getValue());
        }
    }

    // TODO: make sure input file(s), libs, etc. actually exist!
    // Ensure our input path exists

    Path p = new Path(props.getProperty(ConfigFields.APP_INPUT_PATH));
    FileSystem fs = FileSystem.get(conf);

    if (!fs.exists(p))
        throw new FileNotFoundException("Input path not found: " + p.toString() + " (in " + fs.getUri() + ")");

    LOG.info("Using input path: " + p.toString());

    // Connect
    ResourceManagerHandler rmHandler = new ResourceManagerHandler(conf, null);
    rmHandler.getClientResourceManager();

    // Create an Application request/ID
    ApplicationId appId = rmHandler.getApplicationId(); // Our AppId
    String appName = props.getProperty(ConfigFields.APP_NAME, ConfigFields.DEFAULT_APP_NAME).replace(' ', '_');

    LOG.info("Got an application, id=" + appId + ", appName=" + appName);

    // Copy resources to [HD]FS
    LOG.debug("Copying resources to filesystem");
    Utils.copyLocalResourcesToFs(props, conf, appId, appName); // Local resources
    Utils.copyLocalResourceToFs(configFile, ConfigFields.APP_CONFIG_FILE, conf, appId, appName); // Config file

    try {
        Utils.copyLocalResourceToFs("log4j.properties", "log4j.properties", conf, appId, appName); // Log4j
    } catch (FileNotFoundException ex) {
        LOG.warn("log4j.properties file not found");
    }

    // Create our context
    List<String> commands = Utils.getMasterCommand(conf, props);
    Map<String, LocalResource> localResources = Utils.getLocalResourcesForApplication(conf, appId, appName,
            props, LocalResourceVisibility.APPLICATION);

    // Submit app
    rmHandler.submitApplication(appId, appName, Utils.getEnvironment(conf, props), localResources, commands,
            Integer.parseInt(props.getProperty(ConfigFields.YARN_MEMORY, "512")));

    /*
     * TODO:
     * - look at updating this code region to make sure job is submitted!
     * 
     */

    StopWatch watch = new StopWatch();
    watch.start();

    // Wait for app to complete
    while (true) {
        Thread.sleep(2000);

        ApplicationReport report = rmHandler.getApplicationReport(appId);
        LOG.info("IterativeReduce report: " + " appId=" + appId.getId() + ", state: "
                + report.getYarnApplicationState().toString() + ", Running Time: " + watch.toString());

        //report.getDiagnostics()

        if (YarnApplicationState.FINISHED == report.getYarnApplicationState()) {
            LOG.info("Application finished in " + (System.currentTimeMillis() - startTime) + "ms");

            if (FinalApplicationStatus.SUCCEEDED == report.getFinalApplicationStatus()) {
                LOG.info("Application completed succesfully.");
                return 0;
            } else {
                LOG.info("Application completed with en error: " + report.getDiagnostics());
                return -1;
            }
        } else if (YarnApplicationState.FAILED == report.getYarnApplicationState()
                || YarnApplicationState.KILLED == report.getYarnApplicationState()) {

            LOG.info("Application completed with a failed or killed state: " + report.getDiagnostics());
            return -1;
        }

    }

}

---

From source file:org.dknight.app.UnmanagedAMLauncher.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if time
 * expires./*from w ww  .ja v  a 2s .com*/
 *
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws YarnException, IOException {

    long foundAMCompletedTime = 0;
    final int timeToWaitMS = 10000;
    StringBuilder expectedFinalState = new StringBuilder();
    boolean first = true;
    for (YarnApplicationState state : finalState) {
        if (first) {
            first = false;
            expectedFinalState.append(state.name());
        } else {
            expectedFinalState.append("," + state.name());
        }
    }

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = rmClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }

        // wait for 10 seconds after process has completed for app report to
        // come back
        if (amCompleted) {
            if (foundAMCompletedTime == 0) {
                foundAMCompletedTime = System.currentTimeMillis();
            } else if ((System.currentTimeMillis() - foundAMCompletedTime) > timeToWaitMS) {
                LOG.warn("Waited " + timeToWaitMS / 1000 + " seconds after process completed for AppReport"
                        + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state
                        + ", ExpectedStates = " + expectedFinalState.toString());
                throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
                        + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
            }
        }
    }
}

---

From source file:org.hdl.caffe.yarn.app.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 *
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException//from  w  w  w  .j av a 2 s . c  o  m
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", caffeAppFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus caffeStatus = report.getFinalApplicationStatus();

        if (YarnApplicationState.RUNNING == state) {
            if (appRpc == null) {
                String hostname = report.getHost();
                int port = report.getRpcPort();
                LOG.info("Application master rpc host: " + hostname + "; port: " + port);
                appRpc = new CaffeApplicationRpcClient(hostname, port).getRpc();
            }
        }

        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == caffeStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", appFinalState=" + caffeStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", appFinalState="
                    + caffeStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }

}

---

From source file:org.hortonworks.dovetail.client.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires./*  w w  w.  j a  va  2s .  c o  m*/
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.finest("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dovetailStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dovetailStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DovetailFinalStatus=" + dovetailStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DovetailFinalStatus="
                    + dovetailStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }
}

---

From source file:org.huahinframework.manager.rest.service.ApplicationService.java

License:Apache License

@Path("/list")
@GET//ww  w  .java2 s.c o  m
@Produces(MediaType.APPLICATION_JSON)
public JSONObject list() {
    JSONObject jsonObject = new JSONObject();

    try {
        GetAllApplicationsRequest request = recordFactory.newRecordInstance(GetAllApplicationsRequest.class);
        GetAllApplicationsResponse response = applicationsManager.getAllApplications(request);

        JSONObject appObject = new JSONObject();
        List<JSONObject> apps = new ArrayList<JSONObject>();
        for (ApplicationReport ar : response.getApplicationList()) {
            JSONObject app = new JSONObject();
            app.put(Response.ID, ar.getApplicationId().toString());
            app.put(Response.USER, ar.getUser());
            app.put(Response.NAME, ar.getName());
            app.put(Response.QUEUE, ar.getQueue());
            YarnApplicationState state = ar.getYarnApplicationState();
            app.put(Response.STATE, state);
            app.put(Response.FINAL_STATUS, ar.getFinalApplicationStatus().name());
            String trackingUrl = ar.getTrackingUrl();
            boolean trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
                    || YarnApplicationState.NEW == state || YarnApplicationState.SUBMITTED == state
                    || YarnApplicationState.ACCEPTED == state;
            String trackingUI = trackingUrlIsNotReady ? "UNASSIGNED"
                    : (ar.getFinishTime() == 0 ? "ApplicationMaster" : "History");
            app.put(Response.TRACKING_UI, trackingUI);
            app.put(Response.TRACKING_URL, trackingUrl);
            app.put(Response.DIAGNOSTICS, ar.getDiagnostics());
            app.put(Response.START_TIME, new Date(ar.getStartTime()));
            app.put(Response.FINISHED_TIME, ar.getFinishTime() == 0 ? "" : new Date(ar.getFinishTime()));
            app.put(Response.ELAPSED_TIME,
                    (Times.elapsed(ar.getStartTime(), ar.getFinishTime()) / 1000) + "sec");
            apps.add(app);
        }

        appObject.put(Response.APP, new JSONArray(apps));
        jsonObject.put(Response.APPS, appObject);
    } catch (Exception e) {
        e.printStackTrace();
        log.error(e);
        Map<String, String> status = new HashMap<String, String>();
        status.put(Response.STATUS, e.getMessage());
        jsonObject = new JSONObject(status);
    }

    return jsonObject;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »