Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getClientToAMToken

List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getClientToAMToken

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.api.records.*
  5. ApplicationReport
  6. getClientToAMToken

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getClientToAMToken.

Prototype

@Public
@Stable
public abstract Token getClientToAMToken();

Source Link

Document

Get the client token for communicating with the ApplicationMaster.

Usage

From source file:com.zqh.hadoop.moya.core.yarn.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires.//  ww w . j  a  v  a  2  s.c o m
 *
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws org.apache.hadoop.yarn.exceptions.YarnException
 * @throws java.io.IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }

}

From source file:io.hops.tensorflow.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.//from w w  w .j av  a 2s.co m
 *
 * @param appId
 *     Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
public boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    YarnApplicationState oldState = null;

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);
        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();

        if (oldState != state) {
            LOG.info("Got application report from ASM for" + "\n\t appId=" + appId.getId()
                    + "\n\t clientToAMToken=" + report.getClientToAMToken() + "\n\t appDiagnostics="
                    + report.getDiagnostics() + "\n\t appMasterHost=" + report.getHost() + "\n\t appQueue="
                    + report.getQueue() + "\n\t appMasterRpcPort=" + report.getRpcPort() + "\n\t appStartTime="
                    + report.getStartTime() + "\n\t yarnAppState=" + report.getYarnApplicationState().toString()
                    + "\n\t distributedFinalState=" + report.getFinalApplicationStatus().toString()
                    + "\n\t appTrackingUrl=" + report.getTrackingUrl() + "\n\t appUser=" + report.getUser());
            oldState = state;
        } else {
            LOG.info("Got application report from ASM for " + appId + " (state: " + state + ")");
        }

        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }
}

From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java

License:Apache License

/**
 * Assess whether job is already finished/failed and 'done' flag needs to be
 * set, prints progress display for client if all is going well.
 * /*from w w w.  ja v a 2 s.  c  om*/
 * @param report
 *            the application report to assess.
 * @return true if job report indicates the job run is over.
 */
private boolean checkProgress(final ApplicationReport report) {
    YarnApplicationState jobState = report.getYarnApplicationState();

    LOG.info(
            "Got applicaton report for appId={}, state={}, progress={}%, amDiag={}, masterHost={}, masterRpcPort={}, queue={}, startTime={}, clientToken={}, finalState={}, trackingUrl={}, user={}",
            this.appId.getId(), report.getYarnApplicationState().toString(),
            DF.format(report.getProgress() * 100), report.getDiagnostics(), report.getHost(),
            report.getRpcPort(), report.getQueue(), report.getStartTime(), report.getClientToAMToken(),
            report.getFinalApplicationStatus().toString(), report.getTrackingUrl(), report.getUser());
    switch (jobState) {
    case FINISHED:
        LOG.info("Application finished in {} ms", (System.currentTimeMillis() - getStartTime()));
        return true;
    case KILLED:
        LOG.error("{} reports KILLED state, diagnostics show: {}", getAppName(), report.getDiagnostics());
        return true;
    case FAILED:
        LOG.error("{} reports FAILED state, diagnostics show: {}", getAppName(), report.getDiagnostics());
        return true;
    default:
        if (this.reportCounter++ % 5 == 0) {
            displayJobReport(report);
        }
        return false;
    }
}

From source file:org.apache.helix.provisioning.yarn.AppLauncher.java

License:Apache License

/**
 * TODO: kill the app only in dev mode. In prod, its ok for the app to continue running if the
 * launcher dies after launching/* w  w  w  .  j a  va  2  s.c o  m*/
 */

private String generateReport(ApplicationReport report) {
    return "Got application report from ASM for" + ", appId=" + _appId.getId() + ", clientToAMToken="
            + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost="
            + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
            + report.getYarnApplicationState().toString() + ", distributedFinalState="
            + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser();
}

From source file:org.apache.hoya.yarn.appmaster.rpc.RpcBinder.java

License:Apache License

public static HoyaClusterProtocol getProxy(final Configuration conf, ApplicationReport application,
        final int rpcTimeout) throws IOException, HoyaException, InterruptedException {

    String host = application.getHost();
    int port = application.getRpcPort();
    String address = host + ":" + port;
    if (host == null || 0 == port) {
        throw new HoyaException(HoyaExitCodes.EXIT_CONNECTIVITY_PROBLEM,
                "Hoya YARN instance " + application.getName() + " isn't providing a valid address for the"
                        + " Hoya RPC protocol: " + address);
    }//  w  w  w  . ja v  a2  s  .  com

    UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
    final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(currentUser.getUserName());
    final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(),
            application.getRpcPort());
    HoyaClusterProtocol realProxy;

    log.debug("Connecting to {}", serviceAddr);
    if (UserGroupInformation.isSecurityEnabled()) {
        org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken();
        Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
        newUgi.addToken(token);
        realProxy = newUgi.doAs(new PrivilegedExceptionAction<HoyaClusterProtocol>() {
            @Override
            public HoyaClusterProtocol run() throws IOException {
                return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
            }
        });
    } else {
        return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
    }
    return realProxy;
}

From source file:org.apache.metron.maas.service.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires./*from   w w w . ja  va  2 s  . c  o m*/
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.RUNNING == state) {
            LOG.info("Application is running...");
            return true;
        }
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

From source file:org.apache.slider.server.appmaster.rpc.RpcBinder.java

License:Apache License

public static SliderClusterProtocol getProxy(final Configuration conf, ApplicationReport application,
        final int rpcTimeout) throws IOException, SliderException, InterruptedException {

    String host = application.getHost();
    int port = application.getRpcPort();
    String address = host + ":" + port;
    if (host == null || 0 == port) {
        throw new SliderException(SliderExitCodes.EXIT_CONNECTIVITY_PROBLEM,
                "Slider instance " + application.getName() + " isn't providing a valid address for the"
                        + " Slider RPC protocol: " + address);
    }//from  ww  w.j  av a2s .c o m

    UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
    final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(currentUser.getUserName());
    final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(),
            application.getRpcPort());
    SliderClusterProtocol realProxy;

    log.debug("Connecting to {}", serviceAddr);
    if (UserGroupInformation.isSecurityEnabled()) {
        org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken();
        Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
        newUgi.addToken(token);
        realProxy = newUgi.doAs(new PrivilegedExceptionAction<SliderClusterProtocol>() {
            @Override
            public SliderClusterProtocol run() throws IOException {
                return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
            }
        });
    } else {
        return connectToServer(serviceAddr, newUgi, conf, rpcTimeout);
    }
    return realProxy;
}

From source file:org.apache.tajo.master.rm.YarnTajoResourceManager.java

License:Apache License

private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws IOException, YarnException {

    long sleepTime = 100;
    int count = 1;
    while (true) {
        // Get application report for the appId we are interested in
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }/*from w  ww .j  ava 2 s.c  o  m*/
        try {
            Thread.sleep(sleepTime);
            sleepTime = count * 100;
            if (count < 10) {
                count++;
            }
        } catch (InterruptedException e) {
            //LOG.debug("Thread sleep in monitoring loop interrupted");
        }
    }
}

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

static DAGClientAMProtocolBlockingPB getSessionAMProxy(FrameworkClient yarnClient, Configuration conf,
        ApplicationId applicationId) throws TezException, IOException {
    ApplicationReport appReport;
    try {//from www .  ja v  a  2 s .c  om
        appReport = yarnClient.getApplicationReport(applicationId);

        if (appReport == null) {
            throw new TezUncheckedException(
                    "Could not retrieve application report" + " from YARN, applicationId=" + applicationId);
        }
        YarnApplicationState appState = appReport.getYarnApplicationState();
        if (appState != YarnApplicationState.RUNNING) {
            if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED
                    || appState == YarnApplicationState.FAILED) {
                String msg = "Application not running" + ", applicationId=" + applicationId
                        + ", yarnApplicationState=" + appReport.getYarnApplicationState()
                        + ", finalApplicationStatus=" + appReport.getFinalApplicationStatus() + ", trackingUrl="
                        + appReport.getTrackingUrl() + ", diagnostics="
                        + (appReport.getDiagnostics() != null ? appReport.getDiagnostics()
                                : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG);
                LOG.info(msg);
                throw new SessionNotRunning(msg);
            }
            return null;
        }
    } catch (YarnException e) {
        throw new TezException(e);
    }
    return getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(), appReport.getClientToAMToken());
}

From source file:org.dknight.app.UnmanagedAMLauncher.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if time
 * expires.//  w  w w.j ava 2s . c o  m
 *
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws YarnException, IOException {

    long foundAMCompletedTime = 0;
    final int timeToWaitMS = 10000;
    StringBuilder expectedFinalState = new StringBuilder();
    boolean first = true;
    for (YarnApplicationState state : finalState) {
        if (first) {
            first = false;
            expectedFinalState.append(state.name());
        } else {
            expectedFinalState.append("," + state.name());
        }
    }

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = rmClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }

        // wait for 10 seconds after process has completed for app report to
        // come back
        if (amCompleted) {
            if (foundAMCompletedTime == 0) {
                foundAMCompletedTime = System.currentTimeMillis();
            } else if ((System.currentTimeMillis() - foundAMCompletedTime) > timeToWaitMS) {
                LOG.warn("Waited " + timeToWaitMS / 1000 + " seconds after process completed for AppReport"
                        + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state
                        + ", ExpectedStates = " + expectedFinalState.toString());
                throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
                        + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
            }
        }
    }
}