List of usage examples for org.apache.hadoop.yarn.api.records Priority newInstance
@Public
@Stable
public static Priority newInstance(int p)
From source file:io.hops.tensorflow.ApplicationMaster.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM *///ww w . j a v a 2 s. c o m public ContainerRequest setupContainerAskForRM(boolean worker) { Priority pri = Priority.newInstance(1); // Set up resource type requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); if (worker) { pri.setPriority(0); // worker: 0, ps: 1 capability.setGPUs(containerGPUs); } ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
From source file:io.hops.tensorflow.Client.java
License:Apache License
private ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication app, ContainerLaunchContext containerContext) { ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); if (name == null) { appContext.setApplicationName(mainRelativePath); } else {// w w w. j a v a 2 s .c om appContext.setApplicationName(name); } appContext.setApplicationType("YARNTF"); appContext.setMaxAppAttempts(maxAppAttempts); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } appContext.setResource(Resource.newInstance(amMemory, amVCores)); appContext.setAMContainerSpec(containerContext); appContext.setPriority(Priority.newInstance(amPriority)); appContext.setQueue(amQueue); // the queue to which this application is to be submitted in the RM return appContext; }
From source file:org.apache.drill.yarn.core.AppSpec.java
License:Apache License
/** * Given this generic description of an application, create the detailed YARN * application submission context required to launch the application. * * @param conf/* w ww. j a v a 2s . c o m*/ * the YARN configuration obtained by reading the Hadoop * configuration files * @param app * the YARN definition of the client application to be populated from * this generic description * @return the completed application launch context for the given application * @throws IOException * if localized resources are not found in the distributed file * system (such as HDFS) */ public ApplicationSubmissionContext createAppLaunchContext(YarnConfiguration conf, YarnClientApplication app) throws IOException { ContainerLaunchContext amContainer = createLaunchContext(conf); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(getCapability()); appContext.setQueue(queueName); // queue appContext.setPriority(Priority.newInstance(priority)); if (!DoYUtil.isBlank(nodeLabelExpr)) { LOG.info("Requesting to run the AM using node expression: " + nodeLabelExpr); appContext.setNodeLabelExpression(nodeLabelExpr); } appContext.setUnmanagedAM(unmanaged); // Only try the AM once. It will fail if things are misconfigured. Retrying // is unlikely // to fix the configuration problem. appContext.setMaxAppAttempts(1); // TODO: Security tokens return appContext; }
From source file:org.apache.flink.yarn.YarnFlinkResourceManager.java
License:Apache License
@Override protected void requestNewWorkers(int numWorkers) { final long mem = taskManagerParameters.taskManagerTotalMemoryMB(); final int containerMemorySizeMB; if (mem <= Integer.MAX_VALUE) { containerMemorySizeMB = (int) mem; } else {//from ww w . j a v a 2s . com containerMemorySizeMB = Integer.MAX_VALUE; LOG.error("Decreasing container size from {} MB to {} MB (integer value overflow)", mem, containerMemorySizeMB); } for (int i = 0; i < numWorkers; i++) { numPendingContainerRequests++; LOG.info("Requesting new TaskManager container with {} megabytes memory. Pending requests: {}", containerMemorySizeMB, numPendingContainerRequests); // Priority for worker containers - priorities are intra-application Priority priority = Priority.newInstance(0); // Resource requirements for worker containers int taskManagerSlots = Integer.valueOf(System.getenv(YarnConfigKeys.ENV_SLOTS)); int vcores = config.getInteger(ConfigConstants.YARN_VCORES, Math.max(taskManagerSlots, 1)); Resource capability = Resource.newInstance(containerMemorySizeMB, vcores); resourceManagerClient .addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority)); } // make sure we transmit the request fast and receive fast news of granted allocations resourceManagerClient.setHeartbeatInterval(FAST_YARN_HEARTBEAT_INTERVAL_MS); }
From source file:org.apache.flink.yarn.YarnResourceManager.java
License:Apache License
@Override public void startNewWorker(ResourceProfile resourceProfile) { // Priority for worker containers - priorities are intra-application //TODO: set priority according to the resource allocated Priority priority = Priority.newInstance(generatePriority(resourceProfile)); int mem = resourceProfile.getMemoryInMB() < 0 ? DEFAULT_TSK_EXECUTOR_MEMORY_SIZE : (int) resourceProfile.getMemoryInMB(); int vcore = resourceProfile.getCpuCores() < 1 ? 1 : (int) resourceProfile.getCpuCores(); Resource capability = Resource.newInstance(mem, vcore); requestYarnContainer(capability, priority); }
From source file:org.apache.gobblin.yarn.GobblinYarnAppLauncher.java
License:Apache License
/** * Setup and submit the Gobblin Yarn application. * * @throws IOException if there's anything wrong setting up and submitting the Yarn application * @throws YarnException if there's anything wrong setting up and submitting the Yarn application *///from w w w .jav a 2s .c o m @VisibleForTesting ApplicationId setupAndSubmitApplication() throws IOException, YarnException { YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); ApplicationId applicationId = appSubmissionContext.getApplicationId(); GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse(); // Set up resource type requirements for ApplicationMaster Resource resource = prepareContainerResource(newApplicationResponse); // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId); ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext .setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory()))); Map<ApplicationAccessType, String> acls = new HashMap<>(1); acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl); amContainerLaunchContext.setApplicationACLs(acls); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); } // Setup the application submission context appSubmissionContext.setApplicationName(this.applicationName); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext); // Also setup container local resources by copying local jars and files the container need to HDFS addContainerLocalResources(applicationId); // Submit the application LOGGER.info("Submitting application " + applicationId); this.yarnClient.submitApplication(appSubmissionContext); LOGGER.info("Application successfully submitted and accepted"); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue()); return applicationId; }
From source file:org.apache.gobblin.yarn.YarnServiceTest.java
License:Apache License
private void startApp() throws Exception { // submit a dummy app ApplicationSubmissionContext appSubmissionContext = yarnClient.createApplication() .getApplicationSubmissionContext(); this.applicationId = appSubmissionContext.getApplicationId(); ContainerLaunchContext containerLaunchContext = BuilderUtils.newContainerLaunchContext( Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap()); // Setup the application submission context appSubmissionContext.setApplicationName("TestApp"); appSubmissionContext.setResource(Resource.newInstance(128, 1)); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(containerLaunchContext); this.yarnClient.submitApplication(appSubmissionContext); // wait for application to be accepted int i;// ww w. j a v a 2 s .co m RMAppAttempt attempt = null; for (i = 0; i < 120; i++) { ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { this.applicationAttemptId = appReport.getCurrentApplicationAttemptId(); attempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt(); break; } Thread.sleep(1000); } Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state"); // Set the AM-RM token in the UGI for access during testing UserGroupInformation.setLoginUser( UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken()); }
From source file:org.apache.hama.bsp.ApplicationMaster.java
License:Apache License
/** * Setup the request that will be sent to the RM for the container ask. * //from w ww . j a v a 2s .c om * @return the setup ResourceRequest to be sent to RM */ private AMRMClient.ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request // TODO - what is the range for priority? how to decide? Priority pri = Priority.newInstance(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu // requirements Resource capability = Resource.newInstance(containerMemory, containerVirtualCores); AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
From source file:org.apache.ignite.yarn.IgniteApplicationMasterSelfTest.java
License:Apache License
/** * @param host Host.// w w w .j a v a 2 s . co m * @param cpu Cpu count. * @param mem Memory. * @return Container. */ private Container createContainer(String host, int cpu, int mem) { return Container.newInstance( ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0), ThreadLocalRandom.current().nextLong()), NodeId.newInstance(host, 0), "example.com", new MockResource(mem, cpu), Priority.newInstance(0), null); }
From source file:org.apache.metron.maas.service.callback.ContainerRequestListener.java
License:Apache License
public void requestContainers(int number, Resource characteristic) { Priority pri = Priority.newInstance(0); state.getQueue(characteristic);/*w ww . j a v a 2s .c o m*/ AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(characteristic, null, null, pri, true); for (int i = 0; i < number; ++i) { amRMClient.addContainerRequest(request); } }