Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getQueue

List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getQueue

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ApplicationReport
  6. getQueue

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getQueue.

Prototype

@Public
@Stable
public abstract String getQueue();

Source Link

Document

Get the queue to which the application was submitted.

Usage

From source file:com.zqh.hadoop.moya.core.yarn.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires./*from  ww  w. ja v a2 s.  c  om*/
 *
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws org.apache.hadoop.yarn.exceptions.YarnException
 * @throws java.io.IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }

}

From source file:gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 *//* www. j a va  2s  . c o m*/
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
    YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
    ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
    appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
    ApplicationId applicationId = appSubmissionContext.getApplicationId();

    GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
    // Set up resource type requirements for ApplicationMaster
    Resource resource = prepareContainerResource(newApplicationResponse);

    // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
    Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

    ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
    amContainerLaunchContext.setLocalResources(appMasterLocalResources);
    amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
    amContainerLaunchContext
            .setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory())));
    if (UserGroupInformation.isSecurityEnabled()) {
        setupSecurityTokens(amContainerLaunchContext);
    }

    // Setup the application submission context
    appSubmissionContext.setApplicationName(this.applicationName);
    appSubmissionContext.setResource(resource);
    appSubmissionContext.setQueue(this.appQueueName);
    appSubmissionContext.setPriority(Priority.newInstance(0));
    appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);

    // Also setup container local resources by copying local jars and files the container need to HDFS
    addContainerLocalResources(applicationId);

    // Submit the application
    LOGGER.info("Submitting application " + applicationId);
    this.yarnClient.submitApplication(appSubmissionContext);

    LOGGER.info("Application successfully submitted and accepted");
    ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
    LOGGER.info("Application Name: " + applicationReport.getName());
    LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
    LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

    return applicationId;
}

From source file:io.hops.tensorflow.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.//from   w  w  w  .j  a  v a  2  s . c  o  m
 *
 * @param appId
 *     Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
public boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    YarnApplicationState oldState = null;

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);
        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();

        if (oldState != state) {
            LOG.info("Got application report from ASM for" + "\n\t appId=" + appId.getId()
                    + "\n\t clientToAMToken=" + report.getClientToAMToken() + "\n\t appDiagnostics="
                    + report.getDiagnostics() + "\n\t appMasterHost=" + report.getHost() + "\n\t appQueue="
                    + report.getQueue() + "\n\t appMasterRpcPort=" + report.getRpcPort() + "\n\t appStartTime="
                    + report.getStartTime() + "\n\t yarnAppState=" + report.getYarnApplicationState().toString()
                    + "\n\t distributedFinalState=" + report.getFinalApplicationStatus().toString()
                    + "\n\t appTrackingUrl=" + report.getTrackingUrl() + "\n\t appUser=" + report.getUser());
            oldState = state;
        } else {
            LOG.info("Got application report from ASM for " + appId + " (state: " + state + ")");
        }

        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}

From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java

License:Apache License

/**
 * Assess whether job is already finished/failed and 'done' flag needs to be
 * set, prints progress display for client if all is going well.
 * //ww w.  j av a2 s  . c  om
 * @param report
 *            the application report to assess.
 * @return true if job report indicates the job run is over.
 */
private boolean checkProgress(final ApplicationReport report) {
    YarnApplicationState jobState = report.getYarnApplicationState();

    LOG.info(
            "Got applicaton report for appId={}, state={}, progress={}%, amDiag={}, masterHost={}, masterRpcPort={}, queue={}, startTime={}, clientToken={}, finalState={}, trackingUrl={}, user={}",
            this.appId.getId(), report.getYarnApplicationState().toString(),
            DF.format(report.getProgress() * 100), report.getDiagnostics(), report.getHost(),
            report.getRpcPort(), report.getQueue(), report.getStartTime(), report.getClientToAMToken(),
            report.getFinalApplicationStatus().toString(), report.getTrackingUrl(), report.getUser());
    switch (jobState) {
    case FINISHED:
        LOG.info("Application finished in {} ms", (System.currentTimeMillis() - getStartTime()));
        return true;
    case KILLED:
        LOG.error("{} reports KILLED state, diagnostics show: {}", getAppName(), report.getDiagnostics());
        return true;
    case FAILED:
        LOG.error("{} reports FAILED state, diagnostics show: {}", getAppName(), report.getDiagnostics());
        return true;
    default:
        if (this.reportCounter++ % 5 == 0) {
            displayJobReport(report);
        }
        return false;
    }
}

From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncher.java

License:Apache License

/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 *///from  w  w w .  java 2  s  .  c o  m
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
    YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
    ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
    appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
    ApplicationId applicationId = appSubmissionContext.getApplicationId();

    GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
    // Set up resource type requirements for ApplicationMaster
    Resource resource = prepareContainerResource(newApplicationResponse);

    // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
    Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

    ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
    amContainerLaunchContext.setLocalResources(appMasterLocalResources);
    amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
    amContainerLaunchContext
            .setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory())));

    Map<ApplicationAccessType, String> acls = new HashMap<>(1);
    acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
    amContainerLaunchContext.setApplicationACLs(acls);

    if (UserGroupInformation.isSecurityEnabled()) {
        setupSecurityTokens(amContainerLaunchContext);
    }

    // Setup the application submission context
    appSubmissionContext.setApplicationName(this.applicationName);
    appSubmissionContext.setResource(resource);
    appSubmissionContext.setQueue(this.appQueueName);
    appSubmissionContext.setPriority(Priority.newInstance(0));
    appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
    // Also setup container local resources by copying local jars and files the container need to HDFS
    addContainerLocalResources(applicationId);

    // Submit the application
    LOGGER.info("Submitting application " + applicationId);
    this.yarnClient.submitApplication(appSubmissionContext);

    LOGGER.info("Application successfully submitted and accepted");
    ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
    LOGGER.info("Application Name: " + applicationReport.getName());
    LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
    LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

    return applicationId;
}

From source file:org.apache.helix.provisioning.yarn.AppLauncher.java

License:Apache License

/**
 * TODO: kill the app only in dev mode. In prod, its ok for the app to continue running if the
 * launcher dies after launching//  w ww.j  a v a  2s  .co  m
 */

private String generateReport(ApplicationReport report) {
    return "Got application report from ASM for" + ", appId=" + _appId.getId() + ", clientToAMToken="
            + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost="
            + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
            + report.getYarnApplicationState().toString() + ", distributedFinalState="
            + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser();
}

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.//from ww  w .  j a  va2s .  c o m
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.RUNNING == state) {
            LOG.info("Application is running...");
            return true;
        }
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:org.apache.slider.core.launch.SerializedApplicationReport.java

License:Apache License

public SerializedApplicationReport(ApplicationReport report) {
    this.applicationId = report.getApplicationId().toString();
    this.applicationAttemptId = report.getCurrentApplicationAttemptId().toString();
    this.name = report.getName();
    this.applicationType = report.getApplicationType();
    this.user = report.getUser();
    this.queue = report.getQueue();
    this.host = report.getHost();
    this.rpcPort = report.getRpcPort();
    this.state = report.getYarnApplicationState().toString();
    this.diagnostics = report.getDiagnostics();
    this.startTime = report.getStartTime();
    this.finishTime = report.getFinishTime();
    this.finalStatus = report.getFinalApplicationStatus().toString();
    this.progress = report.getProgress();
}

From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java

License:Apache License

private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws IOException, YarnException {

    long sleepTime = 100;
    int count = 1;
    while (true) {
        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }//from w  w w  .  j a v a2 s .  co  m
        try {
            Thread.sleep(sleepTime);
            sleepTime = count * 100;
            if (count < 10) {
                count++;
            }
        } catch (InterruptedException e) {
            //LOG.debug("Thread sleep in monitoring loop interrupted");
        }
    }
}

From source file:org.dknight.app.UnmanagedAMLauncher.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if time
 * expires.//from   w  w w .  j  a va  2 s.c o  m
 *
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws YarnException, IOException {

    long foundAMCompletedTime = 0;
    final int timeToWaitMS = 10000;
    StringBuilder expectedFinalState = new StringBuilder();
    boolean first = true;
    for (YarnApplicationState state : finalState) {
        if (first) {
            first = false;
            expectedFinalState.append(state.name());
        } else {
            expectedFinalState.append("," + state.name());
        }
    }

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = rmClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }

        // wait for 10 seconds after process has completed for app report to
        // come back
        if (amCompleted) {
            if (foundAMCompletedTime == 0) {
                foundAMCompletedTime = System.currentTimeMillis();
            } else if ((System.currentTimeMillis() - foundAMCompletedTime) > timeToWaitMS) {
                LOG.warn("Waited " + timeToWaitMS / 1000 + " seconds after process completed for AppReport"
                        + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state
                        + ", ExpectedStates = " + expectedFinalState.toString());
                throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
                        + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
            }
        }
    }
}