List of usage examples for org.apache.hadoop.yarn.api.records ApplicationReport getCurrentApplicationAttemptId
@Public @Stable public abstract ApplicationAttemptId getCurrentApplicationAttemptId();
ApplicationAttemptId of the current attempt of the application From source file:UnmanagedAMLauncher.java
License:Apache License
/** * Monitor the submitted application for completion. Kill application if time * expires.// ww w .j a v a 2 s . c o m * * @param appId * Application Id of application to be monitored * @return true if application completed successfully * @throws YarnException * @throws IOException */ private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState) throws YarnException, IOException { long foundAMCompletedTime = 0; StringBuilder expectedFinalState = new StringBuilder(); boolean first = true; for (YarnApplicationState state : finalState) { if (first) { first = false; expectedFinalState.append(state.name()); } else { expectedFinalState.append("," + state.name()); } } while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = rmClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId=" + report.getCurrentApplicationAttemptId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); if (finalState.contains(state)) { return report; } // wait for 10 seconds after process has completed for app report to // come back if (amCompleted) { if (foundAMCompletedTime == 0) { foundAMCompletedTime = System.currentTimeMillis(); } else if ((System.currentTimeMillis() - foundAMCompletedTime) > AM_STATE_WAIT_TIMEOUT_MS) { LOG.warn("Waited " + AM_STATE_WAIT_TIMEOUT_MS / 1000 + " seconds after process completed for AppReport" + " to reach desired final state. Not waiting anymore." + "CurrentState = " + state + ", ExpectedStates = " + expectedFinalState.toString()); throw new RuntimeException("Failed to receive final expected state" + " in ApplicationReport" + ", CurrentState=" + state + ", ExpectedStates=" + expectedFinalState.toString()); } } } }
From source file:com.datatorrent.stram.InlineAM.java
License:Apache License
public boolean run() throws Exception { LOG.info("Starting Client"); // Connect to ResourceManager rmClient.start();//from w w w . j a v a 2s .c o m try { // Get a new application id YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master LOG.info("Setting up application submission context for ASM"); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName(appName); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); // unmanaged AM appContext.setUnmanagedAM(true); LOG.info("Setting unmanaged AM"); // Submit the application to the applications manager LOG.info("Submitting application to ASM"); rmClient.submitApplication(appContext); // Monitor the application to wait for launch state ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching application with id: " + attemptId); // launch AM runAM(attemptId); // Monitor the application for end state appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED)); YarnApplicationState appState = appReport.getYarnApplicationState(); FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus(); LOG.info("App ended with state: " + appReport.getYarnApplicationState() + " and status: " + appStatus); boolean success; if (YarnApplicationState.FINISHED == appState && FinalApplicationStatus.SUCCEEDED == appStatus) { LOG.info("Application has completed successfully."); success = true; } else { LOG.info("Application did finished unsuccessfully." + " YarnState=" + appState.toString() + ", FinalStatus=" + appStatus.toString()); success = false; } return success; } finally { rmClient.stop(); } }
From source file:com.datatorrent.stram.InlineAM.java
License:Apache License
/** * Monitor the submitted application for completion. Kill application if time * expires./*from w ww.j a v a 2 s.com*/ * * @param appId * Application Id of application to be monitored * @return true if application completed successfully * @throws YarnRemoteException */ private ApplicationReport monitorApplication(ApplicationId appId, Set<YarnApplicationState> finalState) throws YarnException, IOException { while (true) { // Check app status every 1 second. try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.debug("Thread sleep in monitoring loop interrupted"); } // Get application report for the appId we are interested in ApplicationReport report = rmClient.getApplicationReport(appId); LOG.info("Got application report from ASM for" + ", appId=" + appId.getId() + ", appAttemptId=" + report.getCurrentApplicationAttemptId() + ", clientToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); if (finalState.contains(state)) { return report; } } }
From source file:com.splicemachine.yarn.test.BareYarnTest.java
License:Apache License
/** * All we really need to do here is to create a yarn client, configure it using the same * yarn-site.xml as was used by the server to start up. * @throws YarnException// w ww. j a v a2 s.c om * @throws IOException */ @Test(timeout = 60000) @Ignore("Broken by dependency change") public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException { // create, submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( Collections.<String, LocalResource>emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>()); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start RMAppAttempt appAttempt; while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnPlatform.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser( UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); }
From source file:ml.shifu.guagua.yarn.GuaguaYarnClient.java
License:Apache License
/** * Display a formatted summary of the job progress report from the AM. * //from ww w . ja va 2s .c om * @param report * the report to display. */ private void displayJobReport(final ApplicationReport report) { if (null == report) { throw new IllegalStateException(String.format( "[*] Latest ApplicationReport for job %s was not received by the local client.", getAppName())); } final float elapsed = (System.currentTimeMillis() - report.getStartTime()) / 1000.0f; LOG.info("{}, Elapsed: {}", getAppName(), String.format("%.2f secs", elapsed)); LOG.info("{}, State: {} , Containers: used/reserved/needed-resources {}/{}/{}", report.getCurrentApplicationAttemptId(), report.getYarnApplicationState().name(), report.getApplicationResourceUsageReport().getNumUsedContainers(), report.getApplicationResourceUsageReport().getNumReservedContainers(), report.getApplicationResourceUsageReport().getNeededResources()); }
From source file:org.apache.drill.yarn.core.YarnRMClient.java
License:Apache License
/** * Waits for the application to start. This version is somewhat informal, the * intended use is when debugging unmanaged applications. * * @throws YarnClientException/*from w w w . ja v a 2 s .c o m*/ */ public ApplicationAttemptId waitForStart() throws YarnClientException { ApplicationReport appReport; YarnApplicationState appState; ApplicationAttemptId attemptId; for (;;) { appReport = getAppReport(); appState = appReport.getYarnApplicationState(); attemptId = appReport.getCurrentApplicationAttemptId(); if (appState != YarnApplicationState.NEW && appState != YarnApplicationState.NEW_SAVING && appState != YarnApplicationState.SUBMITTED) { break; } System.out.println("App State: " + appState); try { Thread.sleep(1000); } catch (InterruptedException e) { // Should never occur. } } if (appState != YarnApplicationState.ACCEPTED) { throw new YarnClientException("Application start failed with status " + appState); } return attemptId; }
From source file:org.apache.flink.yarn.YarnConfigurationITCase.java
License:Apache License
/** * Tests that the Flink components are started with the correct * memory settings.//from w w w . ja v a 2s . c om */ @Test(timeout = 60000) public void testFlinkContainerMemory() throws Exception { final YarnClient yarnClient = getYarnClient(); final Configuration configuration = new Configuration(flinkConfiguration); final int masterMemory = 64; final int taskManagerMemory = 128; final int slotsPerTaskManager = 3; // disable heap cutoff min configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0); configuration.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(1L << 20)); configuration.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(4L << 20)); final YarnConfiguration yarnConfiguration = getYarnConfiguration(); final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(configuration, yarnConfiguration, CliFrontend.getConfigurationDirectoryFromEnv(), yarnClient, true); clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath())); clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles())); final File streamingWordCountFile = getTestJarPath("WindowJoin.jar"); final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile); final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1); try { final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(masterMemory).setTaskManagerMemoryMB(taskManagerMemory) .setSlotsPerTaskManager(slotsPerTaskManager).createClusterSpecification(); final ClusterClient<ApplicationId> clusterClient = clusterDescriptor .deployJobCluster(clusterSpecification, jobGraph, true); final ApplicationId clusterId = clusterClient.getClusterId(); final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor()); try { final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId); final ApplicationAttemptId currentApplicationAttemptId = applicationReport .getCurrentApplicationAttemptId(); // wait until we have second container allocated List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId); while (containers.size() < 2) { // this is nasty but Yarn does not offer a better way to wait Thread.sleep(50L); containers = yarnClient.getContainers(currentApplicationAttemptId); } for (ContainerReport container : containers) { if (container.getContainerId().getId() == 1) { // this should be the application master assertThat(container.getAllocatedResource().getMemory(), is(masterMemory)); } else { assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory)); } } final URI webURI = new URI(clusterClient.getWebInterfaceURL()); CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture; Collection<TaskManagerInfo> taskManagerInfos; while (true) { taskManagersInfoCompletableFuture = restClient.sendRequest(webURI.getHost(), webURI.getPort(), TaskManagersHeaders.getInstance(), EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()); final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get(); taskManagerInfos = taskManagersInfo.getTaskManagerInfos(); // wait until the task manager has registered and reported its slots if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) { break; } else { Thread.sleep(100L); } } // there should be at least one TaskManagerInfo final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next(); assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager)); final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters .create(configuration, taskManagerMemory, slotsPerTaskManager); final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L; // We compare here physical memory assigned to a container with the heap memory that we should pass to // jvm as Xmx parameter. Those value might differ significantly due to sytem page size or jvm // implementation therefore we use 15% threshold here. assertThat((double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize, is(closeTo(1.0, 0.15))); } finally { restClient.shutdown(TIMEOUT); clusterClient.shutdown(); } clusterDescriptor.killCluster(clusterId); } finally { clusterDescriptor.close(); } }
From source file:org.apache.giraph.yarn.GiraphYarnClient.java
License:Apache License
/** * Display a formatted summary of the job progress report from the AM. * @param report the report to display.//from w ww .j av a 2 s . c o m */ private void displayJobReport(final ApplicationReport report) { if (null == report) { throw new IllegalStateException( "[*] Latest ApplicationReport for job " + jobName + " was not received by the local client."); } final float elapsed = (System.currentTimeMillis() - report.getStartTime()) / 1000.0f; LOG.info(jobName + ", Elapsed: " + String.format("%.2f secs", elapsed)); LOG.info(report.getCurrentApplicationAttemptId() + ", State: " + report.getYarnApplicationState().name() + ", Containers used: " + report.getApplicationResourceUsageReport().getNumUsedContainers()); }
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
private void startApp() throws Exception { // submit a dummy app ApplicationSubmissionContext appSubmissionContext = yarnClient.createApplication() .getApplicationSubmissionContext(); this.applicationId = appSubmissionContext.getApplicationId(); ContainerLaunchContext containerLaunchContext = BuilderUtils.newContainerLaunchContext( Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap()); // Setup the application submission context appSubmissionContext.setApplicationName("TestApp"); appSubmissionContext.setResource(Resource.newInstance(128, 1)); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(containerLaunchContext); this.yarnClient.submitApplication(appSubmissionContext); // wait for application to be accepted int i;/*ww w.ja v a 2 s. c o m*/ RMAppAttempt attempt = null; for (i = 0; i < 120; i++) { ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { this.applicationAttemptId = appReport.getCurrentApplicationAttemptId(); attempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt(); break; } Thread.sleep(1000); } Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state"); // Set the AM-RM token in the UGI for access during testing UserGroupInformation.setLoginUser( UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken()); }
From source file:org.apache.reef.runtime.yarn.client.unmanaged.UnmanagedAmYarnSubmissionHelper.java
License:Apache License
@Override public void close() { if (LOG.isLoggable(Level.FINER)) { try {/*from ww w . j a v a2 s .co m*/ final ApplicationReport appReport = this.yarnClient.getApplicationReport(this.applicationId); LOG.log(Level.FINER, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { this.applicationId, appReport.getCurrentApplicationAttemptId(), appReport.getYarnApplicationState(), appReport.getFinalApplicationStatus() }); } catch (final IOException | YarnException ex) { LOG.log(Level.WARNING, "Cannot get final status of Unmanaged AM app: " + this.applicationId, ex); } } LOG.log(Level.FINE, "Closing Unmanaged AM YARN application: {0}", this.applicationId); this.yarnClient.stop(); }