Example usage for org.apache.hadoop.yarn.api.records ApplicationReport getRpcPort

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.api.records ApplicationReport getRpcPort.

Prototype

@Public
@Stable
public abstract int getRpcPort();

Source Link

Document

Get the RPC port of the ApplicationMaster.

Usage

From source file:org.apache.tez.client.TezClientUtils.java

License:Apache License

static DAGClientAMProtocolBlockingPB getSessionAMProxy(FrameworkClient yarnClient, Configuration conf,
        ApplicationId applicationId) throws TezException, IOException {
    ApplicationReport appReport;
    try {//w w  w  .ja  va  2 s. c  o m
        appReport = yarnClient.getApplicationReport(applicationId);

        if (appReport == null) {
            throw new TezUncheckedException(
                    "Could not retrieve application report" + " from YARN, applicationId=" + applicationId);
        }
        YarnApplicationState appState = appReport.getYarnApplicationState();
        if (appState != YarnApplicationState.RUNNING) {
            if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED
                    || appState == YarnApplicationState.FAILED) {
                String msg = "Application not running" + ", applicationId=" + applicationId
                        + ", yarnApplicationState=" + appReport.getYarnApplicationState()
                        + ", finalApplicationStatus=" + appReport.getFinalApplicationStatus() + ", trackingUrl="
                        + appReport.getTrackingUrl() + ", diagnostics="
                        + (appReport.getDiagnostics() != null ? appReport.getDiagnostics()
                                : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG);
                LOG.info(msg);
                throw new SessionNotRunning(msg);
            }
            return null;
        }
    } catch (YarnException e) {
        throw new TezException(e);
    }
    return getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(), appReport.getClientToAMToken());
}

---

From source file:org.dknight.app.UnmanagedAMLauncher.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if time
 * expires.// w  w w .  j a v  a  2  s  .c o m
 *
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState)
        throws YarnException, IOException {

    long foundAMCompletedTime = 0;
    final int timeToWaitMS = 10000;
    StringBuilder expectedFinalState = new StringBuilder();
    boolean first = true;
    for (YarnApplicationState state : finalState) {
        if (first) {
            first = false;
            expectedFinalState.append(state.name());
        } else {
            expectedFinalState.append("," + state.name());
        }
    }

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in
        ApplicationReport report = rmClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId="
                + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken()
                + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
                + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort()
                + ", appStartTime=" + report.getStartTime() + ", yarnAppState="
                + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        if (finalState.contains(state)) {
            return report;
        }

        // wait for 10 seconds after process has completed for app report to
        // come back
        if (amCompleted) {
            if (foundAMCompletedTime == 0) {
                foundAMCompletedTime = System.currentTimeMillis();
            } else if ((System.currentTimeMillis() - foundAMCompletedTime) > timeToWaitMS) {
                LOG.warn("Waited " + timeToWaitMS / 1000 + " seconds after process completed for AppReport"
                        + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state
                        + ", ExpectedStates = " + expectedFinalState.toString());
                throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport"
                        + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString());
            }
        }
    }
}

---

From source file:org.hdl.caffe.yarn.app.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 *
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException//from w  ww  . j  av a  2 s  .c o  m
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", caffeAppFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus caffeStatus = report.getFinalApplicationStatus();

        if (YarnApplicationState.RUNNING == state) {
            if (appRpc == null) {
                String hostname = report.getHost();
                int port = report.getRpcPort();
                LOG.info("Application master rpc host: " + hostname + "; port: " + port);
                appRpc = new CaffeApplicationRpcClient(hostname, port).getRpc();
            }
        }

        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == caffeStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", appFinalState=" + caffeStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", appFinalState="
                    + caffeStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }

}

---

From source file:org.hdl.tensorflow.yarn.client.Client.java

License:Apache License

static ClusterSpec getClusterSpec(YarnClient client, ApplicationId appId) throws Exception {
    ClusterSpec clusterSpec = ClusterSpec.empty();
    ApplicationReport report = client.getApplicationReport(appId);
    YarnApplicationState state = report.getYarnApplicationState();
    if (state.equals(YarnApplicationState.RUNNING)) {
        String hostname = report.getHost();
        int port = report.getRpcPort();
        TFApplicationRpc rpc = TFApplicationRpcClient.getInstance(hostname, port);
        String spec = rpc.getClusterSpec();
        if (spec != null) {
            clusterSpec = ClusterSpec.fromJsonString(spec);
        }//  w  w w . j a  va  2s  .co m
    }
    return clusterSpec;
}

---

From source file:org.hortonworks.dovetail.client.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. Kill application if
 * time expires./*  www.  ja  v  a2s .co  m*/
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.finest("Thread sleep in monitoring loop interrupted");
        }

        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dovetailStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dovetailStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DovetailFinalStatus=" + dovetailStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DovetailFinalStatus="
                    + dovetailStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }
}

---

From source file:org.springframework.yarn.examples.ComplexExampleTests.java

License:Apache License

@Test
@Timed(millis = 360000)/*from   w w  w.  j  a  v  a2s  .  co m*/
public void testAppSubmission() throws Exception {
    // submit and wait running state
    ApplicationId applicationId = submitApplication();
    assertNotNull(applicationId);
    YarnApplicationState state = waitState(applicationId, 120, TimeUnit.SECONDS, YarnApplicationState.RUNNING);
    assertNotNull(state);
    assertTrue(state.equals(YarnApplicationState.RUNNING));

    // check registered rpc port
    ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId);
    String rpcHost = applicationReport.getHost();
    int rpcPort = applicationReport.getRpcPort();
    assertThat(rpcHost, notNullValue());
    assertThat(rpcPort, greaterThan(0));

    File baseDir = getYarnCluster().getYarnWorkDir();
    String baseUrl = findXdBaseUrl(applicationId);

    // we're starting for zero, launch and wait
    setContainerCountViaThrift(1, "default", rpcHost, rpcPort);
    setContainerCountViaThrift(1, "xdgroup", rpcHost, rpcPort);
    int count = ApplicationTestUtils.waitResourcesMatchCount(baseDir, PAT_C_STDOUT, 2, true, null);
    assertThat(count, is(2));

    // do ticktock request and wait logging message
    StreamDefinitionResource stream = createTickTockStream(baseUrl);
    // for some reason stream name is null, bug in xd?
    //assertThat(stream.getName(), is("ticktock"));
    count = ApplicationTestUtils.waitResourcesMatchCount(baseDir, PAT_C_STDOUT, 1, true, "LoggingHandler");
    assertThat(count, is(1));

    // resize and wait
    setContainerCountViaThrift(2, "default", rpcHost, rpcPort);
    setContainerCountViaThrift(2, "xdgroup", rpcHost, rpcPort);
    count = ApplicationTestUtils.waitResourcesMatchCount(baseDir, PAT_C_STDOUT, 4, true, null);
    assertThat(count, is(4));

    deleteTickTockStream(baseUrl);

    // long running app, kill it and check that state is KILLED
    killApplication(applicationId);
    state = waitState(applicationId, 30, TimeUnit.SECONDS, YarnApplicationState.KILLED);
    assertThat(state, is(YarnApplicationState.KILLED));

    // appmaster and 4 container should make it 10 log files
    Resource[] resources = ApplicationTestUtils.matchResources(baseDir, PAT_ALL);
    assertThat(resources, notNullValue());
    assertThat(resources.length, is(10));
}

---

From source file:org.starschema.hadoop.yarn.applications.distributedshell.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. //w  w w  . j  a  v  a  2 s. c  om
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        //      if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
        //        LOG.info("Reached client specified timeout for application. Killing application");
        //        forceKillApplication(appId);
        //        return false;            
        //      }
    }

}

---

From source file:proxyyarn.ProxyYarn.java

License:Apache License

private boolean monitorApplication(YarnClient yarnClient, ApplicationId appId)
        throws YarnException, IOException {

    long clientStartTime = System.currentTimeMillis();
    while (true) {

        // Check app status every 1 second.
        try {//  www  .  java 2  s.com
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            log.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        log.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                log.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                log.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            log.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            log.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(yarnClient, appId);
            return false;
        }
    }
}

---

From source file:tachyon.yarn.Client.java

License:Apache License

/**
 * Monitors the submitted application for completion.
 *
 * @return true if application completed successfully
 * @throws YarnException if errors occur when obtaining application report from ResourceManager
 * @throws IOException if errors occur when obtaining application report from ResourceManager
 *///ww  w  .jav  a 2 s.c o m
private boolean monitorApplication() throws YarnException, IOException {
    DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
    while (true) {
        // Check app status every 10 second.
        CommonUtils.sleepMs(10 * Constants.SECOND_MS);
        Date date = new Date();
        String timeDateStr = dateFormat.format(date);

        // Get application report for the appId we are interested in
        ApplicationReport report = mYarnClient.getApplicationReport(mAppId);

        System.out.println(timeDateStr + " Got application report from ASM for appId=" + mAppId.getId()
                + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics="
                + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue="
                + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime="
                + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString()
                + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
                + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                System.out.println("Application has completed successfully. Breaking monitoring loop");
                return true;
            }
            System.out.println("Application did finished unsuccessfully. YarnState=" + state.toString()
                    + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            System.out.println("Application did not finish. YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }
    }
}

---

From source file:timo.yarn_app_call_java_app.Client.java

License:Apache License

/**
 * Monitor the submitted application for completion.
 * Kill application if time expires./*from   w w  w  .  ja v  a  2 s .c  o  m*/
 * 
 * @param appId
 *          Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {

    while (true) {

        // Check app status every 1 second.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.debug("Thread sleep in monitoring loop interrupted");
        }

        // Get application report for the appId we are interested in 
        ApplicationReport report = yarnClient.getApplicationReport(appId);

        LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken="
                + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics()
                + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue()
                + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime()
                + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState="
                + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl()
                + ", appUser=" + report.getUser());

        YarnApplicationState state = report.getYarnApplicationState();
        FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
        if (YarnApplicationState.FINISHED == state) {
            if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
                LOG.info("Application has completed successfully. Breaking monitoring loop");
                return true;
            } else {
                LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
                        + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
                return false;
            }
        } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
            LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus="
                    + dsStatus.toString() + ". Breaking monitoring loop");
            return false;
        }

        // Client timeout
        if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
            LOG.info("Reached client specified timeout for application. Killing application");
            forceKillApplication(appId);
            return false;
        }
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• »