List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getHost
@Public @Stable public abstract String getHost();
ApplicationMaster is running. From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
public HelixConnection pollForConnection() { String prevReport = ""; HelixConnection connection = null;/*from w w w . j a va 2s.c o m*/ while (true) { try { // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(_appId); String reportMessage = generateReport(report); if (!reportMessage.equals(prevReport)) { LOG.info(reportMessage); } YarnApplicationState state = report.getYarnApplicationState(); if (YarnApplicationState.RUNNING == state) { if (connection == null) { String hostName = null; int ind = report.getHost().indexOf('/'); if (ind > -1) { hostName = report.getHost().substring(ind + 1); } else { hostName = report.getHost(); } connection = new ZkHelixConnection(hostName + ":2181"); try { connection.connect(); } catch (Exception e) { LOG.warn("AppMaster started but not yet initialized"); connection = null; } } if (connection.isConnected()) { return connection; } } prevReport = reportMessage; Thread.sleep(10000); } catch (Exception e) { LOG.error("Exception while getting info", e); break; } } return null; }
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
/** * @return true if successfully completed, it will print status every X seconds *///w w w .j a v a 2 s . c o m public boolean waitUntilDone() { String prevReport = ""; HelixConnection connection = null; while (true) { try { // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(_appId); String reportMessage = generateReport(report); if (!reportMessage.equals(prevReport)) { LOG.info(reportMessage); } YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (YarnApplicationState.RUNNING == state) { if (connection == null) { String hostName = null; int ind = report.getHost().indexOf('/'); if (ind > -1) { hostName = report.getHost().substring(ind + 1); } else { hostName = report.getHost(); } connection = new ZkHelixConnection(hostName + ":2181"); try { connection.connect(); } catch (Exception e) { LOG.warn("AppMaster started but not yet initialized"); connection = null; } } if (connection.isConnected()) { AppStatusReportGenerator generator = new AppStatusReportGenerator(); ClusterId clusterId = ClusterId.from(_applicationSpec.getAppName()); String generateReport = generator.generateReport(connection, clusterId); LOG.info(generateReport); } } prevReport = reportMessage; Thread.sleep(10000); } catch (Exception e) { LOG.error("Exception while getting info", e); break; } } return true; }
From source file:org.apache.helix.provisioning.yarn.AppLauncher.java
License:Apache License
/** * TODO: kill the app only in dev mode. In prod, its ok for the app to continue running if the * launcher dies after launching//ww w.j av a 2 s.c o m */ private String generateReport(ApplicationReport report) { return "Got application report from ASM for" + ", appId=" + _appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser(); }
From source file:org.apache.hoya.tools.HoyaUtils.java
License:Apache License
public static String appReportToString(ApplicationReport r, String separator) { StringBuilder builder = new StringBuilder(512); builder.append("application ").append(r.getName()).append("/").append(r.getApplicationType()); builder.append(separator).append("state: ").append(r.getYarnApplicationState()); builder.append(separator).append("URL: ").append(r.getTrackingUrl()); builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString()); long finishTime = r.getFinishTime(); if (finishTime > 0) { builder.append(separator).append("Finished ").append(new Date(finishTime).toGMTString()); }//from w w w . j a v a 2s. c om builder.append(separator).append("RPC :").append(r.getHost()).append(':').append(r.getRpcPort()); String diagnostics = r.getDiagnostics(); if (!diagnostics.isEmpty()) { builder.append(separator).append("Diagnostics :").append(diagnostics); } return builder.toString(); }
From source file:org.apache.hoya.tools.HoyaUtils.java
License:Apache License
/** * convert an AM report to a string for diagnostics * @param report the report/*from w w w .j a v a2 s. c o m*/ * @return the string value */ public static String reportToString(ApplicationReport report) { if (report == null) { return "Null application report"; } return "App " + report.getName() + "/" + report.getApplicationType() + "# " + report.getApplicationId() + " user " + report.getUser() + " is in state " + report.getYarnApplicationState() + "RPC: " + report.getHost() + ":" + report.getRpcPort(); }
From source file:org.apache.hoya.yarn.appmaster.rpc.RpcBinder.java
License:Apache License
public static HoyaClusterProtocol getProxy(final Configuration conf, ApplicationReport application, final int rpcTimeout) throws IOException, HoyaException, InterruptedException { String host = application.getHost(); int port = application.getRpcPort(); String address = host + ":" + port; if (host == null || 0 == port) { throw new HoyaException(HoyaExitCodes.EXIT_CONNECTIVITY_PROBLEM, "Hoya YARN instance " + application.getName() + " isn't providing a valid address for the" + " Hoya RPC protocol: " + address); }// www . j a v a2 s.c o m UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); final UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(currentUser.getUserName()); final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(application.getHost(), application.getRpcPort()); HoyaClusterProtocol realProxy; log.debug("Connecting to {}", serviceAddr); if (UserGroupInformation.isSecurityEnabled()) { org.apache.hadoop.yarn.api.records.Token clientToAMToken = application.getClientToAMToken(); Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr); newUgi.addToken(token); realProxy = newUgi.doAs(new PrivilegedExceptionAction<HoyaClusterProtocol>() { @Override public HoyaClusterProtocol run() throws IOException { return connectToServer(serviceAddr, newUgi, conf, rpcTimeout); } }); } else { return connectToServer(serviceAddr, newUgi, conf, rpcTimeout); } return realProxy; }
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
/** * Monitor the submitted application for completion. * Kill application if time expires.// w ww .j a va 2s.c o m * @param appId Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = yarnClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.RUNNING == state) { LOG.info("Application is running..."); return true; } if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) { LOG.info("Application has completed successfully. Breaking monitoring loop"); return true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) { LOG.info("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop"); return false; } if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { LOG.info("Reached client specified timeout for application. Killing application"); forceKillApplication(appId); return false; } } }
From source file:org.apache.metron.maas.service.MaasIntegrationTest.java
License:Apache License
public void testDSShell(boolean haveDomain) throws Exception { MaaSConfig config = new MaaSConfig() { {//from w ww . j a va 2 s.c o m setServiceRoot("/maas/service"); setQueueConfig(new HashMap<String, Object>() { { put(ZKQueue.ZK_PATH, "/maas/queue"); } }); } }; String configRoot = "/maas/config"; byte[] configData = ConfigUtil.INSTANCE.toBytes(config); try { client.setData().forPath(configRoot, configData); } catch (KeeperException.NoNodeException e) { client.create().creatingParentsIfNeeded().forPath(configRoot, configData); } String[] args = { "--jar", yarnComponent.getAppMasterJar(), "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--master_memory", "512", "--master_vcores", "2", }; if (haveDomain) { String[] domainArgs = { "--domain", "TEST_DOMAIN", "--view_acls", "reader_user reader_group", "--modify_acls", "writer_user writer_group", "--create" }; List<String> argsList = new ArrayList<String>(Arrays.asList(args)); argsList.addAll(Arrays.asList(domainArgs)); args = argsList.toArray(new String[argsList.size()]); } YarnConfiguration conf = yarnComponent.getConfig(); LOG.info("Initializing DS Client"); final Client client = new Client(new Configuration(conf)); boolean initSuccess = client.init(args); Assert.assertTrue(initSuccess); LOG.info("Running DS Client"); final AtomicBoolean result = new AtomicBoolean(false); Thread t = new Thread() { @Override public void run() { try { result.set(client.run()); } catch (Exception e) { throw new RuntimeException(e); } } }; t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); boolean verified = false; String errorMessage = ""; while (!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0) { Thread.sleep(10); continue; } ApplicationReport appReport = apps.get(0); if (appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; } errorMessage = "Expected host name to start with '" + hostName + "', was '" + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break; } } Assert.assertTrue(errorMessage, verified); FileSystem fs = FileSystem.get(conf); try { new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--local_model_path", "src/test/resources/maas", "--hdfs_model_path", new Path(fs.getHomeDirectory(), "maas/dummy").toString(), "--num_instances", "1", "--memory", "100", "--mode", "ADD", "--log4j", "src/test/resources/log4j.properties" }); ServiceDiscoverer discoverer = new ServiceDiscoverer(this.client, config.getServiceRoot()); discoverer.start(); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); if (endpoints != null && endpoints.size() == 1) { LOG.trace("Found endpoints: " + endpoints.get(0)); String output = makeRESTcall( new URL(endpoints.get(0).getEndpoint().getUrl() + "/echo/casey")); if (output.contains("casey")) { passed = true; break; } } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); Assert.assertNotNull(endpoints); Assert.assertEquals(1, endpoints.size()); } new ModelSubmission().execute(FileSystem.get(conf), new String[] { "--name", "dummy", "--version", "1.0", "--zk_quorum", zkServerComponent.getConnectionString(), "--zk_root", configRoot, "--num_instances", "1", "--mode", "REMOVE", }); { boolean passed = false; for (int i = 0; i < 100; ++i) { try { List<ModelEndpoint> endpoints = discoverer.getEndpoints(new Model("dummy", "1.0")); //ensure that the endpoint is dead. if (endpoints == null || endpoints.size() == 0) { passed = true; break; } } catch (Exception e) { } Thread.sleep(2000); } Assert.assertTrue(passed); } } finally { cleanup(); } }
From source file:org.apache.slider.common.tools.SliderUtils.java
License:Apache License
public static String appReportToString(ApplicationReport r, String separator) { StringBuilder builder = new StringBuilder(512); builder.append("application ").append(r.getName()).append("/").append(r.getApplicationType()) .append(separator);// w ww .j a va 2 s .c o m Set<String> tags = r.getApplicationTags(); if (!tags.isEmpty()) { for (String tag : tags) { builder.append(tag).append(separator); } } DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"); dateFormat.setTimeZone(TimeZone.getDefault()); builder.append("state: ").append(r.getYarnApplicationState()); String trackingUrl = r.getTrackingUrl(); if (isSet(trackingUrl)) { builder.append(separator).append("URL: ").append(trackingUrl); } builder.append(separator).append("Started: ").append(dateFormat.format(new Date(r.getStartTime()))); long finishTime = r.getFinishTime(); if (finishTime > 0) { builder.append(separator).append("Finished: ").append(dateFormat.format(new Date(finishTime))); } String rpcHost = r.getHost(); if (!isSet(rpcHost)) { builder.append(separator).append("RPC :").append(rpcHost).append(':').append(r.getRpcPort()); } String diagnostics = r.getDiagnostics(); if (!isSet(diagnostics)) { builder.append(separator).append("Diagnostics :").append(diagnostics); } return builder.toString(); }
From source file:org.apache.slider.common.tools.SliderUtils.java
License:Apache License
/** * convert an AM report to a string for diagnostics * @param report the report/*from w ww . j a v a2s. c o m*/ * @return the string value */ public static String reportToString(ApplicationReport report) { if (report == null) { return "Null application report"; } return "App " + report.getName() + "/" + report.getApplicationType() + "# " + report.getApplicationId() + " user " + report.getUser() + " is in state " + report.getYarnApplicationState() + " RPC: " + report.getHost() + ":" + report.getRpcPort() + " URL" + report.getOriginalTrackingUrl(); }