List of usage examples for org.apache.hadoop.yarn.api.records Priority newInstance
@Public
@Stable
public static Priority newInstance(int p)
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
/** * Main run function for the client//ww w. ja v a 2s .c o m * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } if (domainId != null && domainId.length() > 0 && toCreateDomain) { prepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts if (domainId != null && domainId.length() > 0) { env.put(Constants.TIMELINEDOMAIN, domainId); } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add(ApplicationMaster.AMOptions.toArgs(ApplicationMaster.AMOptions.ZK_QUORUM.of(zkQuorum), ApplicationMaster.AMOptions.ZK_ROOT.of(zkRoot), ApplicationMaster.AMOptions.APP_JAR_PATH.of(ajPath.toString()))); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application return monitorApplication(appId); }
From source file:org.apache.myriad.state.MockRMApp.java
License:Apache License
public MockRMApp(int newId, long time, RMAppState state) { finish = time;/*ww w . j av a 2 s . c o m*/ id = ApplicationId.newInstance(System.currentTimeMillis(), newId); context = ApplicationSubmissionContext.newInstance(id, name, queue, Priority.newInstance(0), null, false, false, newId, null, applicationType); this.state = state; }
From source file:org.apache.reef.runtime.yarn.client.unmanaged.UnmanagedAmYarnSubmissionHelper.java
License:Apache License
/** * Set the priority of the job.//from www . j ava2 s.com * @param priority YARN application priority. * @return reference to self for chain calls. */ UnmanagedAmYarnSubmissionHelper setPriority(final int priority) { this.applicationSubmissionContext.setPriority(Priority.newInstance(priority)); return this; }
From source file:org.apache.reef.runtime.yarn.client.YarnJobSubmissionHandler.java
License:Apache License
private Priority getPriority(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { return Priority.newInstance(jobSubmissionProto.hasPriority() ? jobSubmissionProto.getPriority() : 0); }
From source file:org.apache.reef.runtime.yarn.client.YarnSubmissionHelper.java
License:Apache License
/** * Set the priority of the job./* w w w . jav a 2 s . c o m*/ * @param priority * @return */ public YarnSubmissionHelper setPriority(final int priority) { this.applicationSubmissionContext.setPriority(Priority.newInstance(priority)); return this; }
From source file:org.apache.samza.job.yarn.refactor.YarnClusterResourceManager.java
License:Apache License
/** * Request resources for running container processes. *///w ww . jav a 2 s. c o m @Override public void requestResources(SamzaResourceRequest resourceRequest) { final int DEFAULT_PRIORITY = 0; log.info("Requesting resources on " + resourceRequest.getPreferredHost() + " for container " + resourceRequest.getContainerID()); int memoryMb = resourceRequest.getMemoryMB(); int cpuCores = resourceRequest.getNumCores(); String preferredHost = resourceRequest.getPreferredHost(); Resource capability = Resource.newInstance(memoryMb, cpuCores); Priority priority = Priority.newInstance(DEFAULT_PRIORITY); AMRMClient.ContainerRequest issuedRequest; if (preferredHost.equals("ANY_HOST")) { log.info("Making a request for ANY_HOST " + preferredHost); issuedRequest = new AMRMClient.ContainerRequest(capability, null, null, priority); } else { log.info("Making a preferred host request on " + preferredHost); issuedRequest = new AMRMClient.ContainerRequest(capability, new String[] { preferredHost }, null, priority); } //ensure that updating the state and making the request are done atomically. synchronized (lock) { requestsMap.put(resourceRequest, issuedRequest); amClient.addContainerRequest(issuedRequest); } }
From source file:org.apache.samza.job.yarn.SamzaContainerRequest.java
License:Apache License
public SamzaContainerRequest(int memoryMb, int cpuCores, int priority, int expectedContainerId, String preferredHost) {//ww w .j av a2 s .c om this.capability = Resource.newInstance(memoryMb, cpuCores); this.priority = Priority.newInstance(priority); this.expectedContainerId = expectedContainerId; if (preferredHost == null) { this.preferredHost = ANY_HOST; this.issuedRequest = new AMRMClient.ContainerRequest(capability, null, null, this.priority); } else { this.preferredHost = preferredHost; this.issuedRequest = new AMRMClient.ContainerRequest(capability, new String[] { this.preferredHost }, null, this.priority); } this.requestTimestamp = System.currentTimeMillis(); }
From source file:org.apache.samza.job.yarn.TestYarnClusterResourceManager.java
License:Apache License
@Test public void testErrorInStartContainerShouldUpdateState() { // create mocks final int samzaContainerId = 1; YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class); SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class); Config config = mock(Config.class); AMRMClientAsync asyncClient = mock(AMRMClientAsync.class); YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081); SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class); SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class); NMClientAsync asyncNMClient = mock(NMClientAsync.class); ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class); // start the cluster manager YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient, callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config); yarnAppState.pendingProcessors.put(String.valueOf(samzaContainerId), new YarnContainer(Container.newInstance( ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1), 1), NodeId.newInstance("host1", 8088), "http://host1", Resource.newInstance(1024, 1), Priority.newInstance(1), Token.newInstance("id".getBytes(), "read", "password".getBytes(), "service")))); yarnClusterResourceManager.start();//from ww w . j av a2 s .co m assertEquals(1, yarnAppState.pendingProcessors.size()); yarnClusterResourceManager.onStartContainerError( ContainerId.newContainerId( ApplicationAttemptId.newInstance(ApplicationId.newInstance(10000l, 1), 1), 1), new Exception()); assertEquals(0, yarnAppState.pendingProcessors.size()); verify(callback, times(1)).onStreamProcessorLaunchFailure(anyObject(), any(Exception.class)); }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Request resources for running container processes. *//* w w w . j a va 2 s. c o m*/ @Override public void requestResources(SamzaResourceRequest resourceRequest) { String processorId = resourceRequest.getProcessorId(); String requestId = resourceRequest.getRequestId(); String preferredHost = resourceRequest.getPreferredHost(); int memoryMb = resourceRequest.getMemoryMB(); int cpuCores = resourceRequest.getNumCores(); Resource capability = Resource.newInstance(memoryMb, cpuCores); String nodeLabelsExpression = yarnConfig.getContainerLabel(); AMRMClient.ContainerRequest issuedRequest; /* * Yarn enforces these two checks: * 1. ANY_HOST requests should always be made with relax-locality = true * 2. A request with relax-locality = false should not be in the same priority as another with relax-locality = true * * Since the Samza AM makes preferred-host requests with relax-locality = false, it follows that ANY_HOST requests * should specify a different priority-level. We can safely set priority of preferred-host requests to be higher than * any-host requests since data-locality is critical. */ if (preferredHost.equals("ANY_HOST")) { Priority priority = Priority.newInstance(ANY_HOST_PRIORITY); boolean relaxLocality = true; log.info( "Requesting resources for Processor ID: {} on nodes: {} on racks: {} with capability: {}, priority: {}, relaxLocality: {}, nodeLabelsExpression: {}", processorId, null, null, capability, priority, relaxLocality, nodeLabelsExpression); issuedRequest = new AMRMClient.ContainerRequest(capability, null, null, priority, relaxLocality, nodeLabelsExpression); } else { String[] nodes = { preferredHost }; Priority priority = Priority.newInstance(PREFERRED_HOST_PRIORITY); boolean relaxLocality = false; log.info( "Requesting resources for Processor ID: {} on nodes: {} on racks: {} with capability: {}, priority: {}, relaxLocality: {}, nodeLabelsExpression: {}", processorId, Arrays.toString(nodes), null, capability, priority, relaxLocality, nodeLabelsExpression); issuedRequest = new AMRMClient.ContainerRequest(capability, nodes, null, priority, relaxLocality, nodeLabelsExpression); } // ensure that updating the state and making the request are done atomically. synchronized (lock) { requestsMap.put(resourceRequest, issuedRequest); amClient.addContainerRequest(issuedRequest); } }
From source file:org.apache.slider.providers.agent.TestAgentProviderService.java
License:Apache License
private AgentProviderService prepareProviderServiceForAgentStateTests() throws IOException { ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); Container container = createNiceMock(Container.class); String role = "HBASE_MASTER"; SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); FileSystem mockFs = new MockFileSystem(); expect(sliderFileSystem.getFileSystem()).andReturn(new FilterFileSystem(mockFs)).anyTimes(); expect(sliderFileSystem.createAmResource(anyObject(Path.class), anyObject(LocalResourceType.class))) .andReturn(createNiceMock(LocalResource.class)).anyTimes(); expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes(); expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes(); expect(container.getPriority()).andReturn(Priority.newInstance(1)); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); Configuration conf = new Configuration(); AgentProviderService aps = createAgentProviderService(conf); AgentProviderService mockAps = Mockito.spy(aps); doReturn(access).when(mockAps).getAmState(); CommandScript cs = new CommandScript(); cs.setScript("scripts/hbase_master.py"); doReturn(cs).when(mockAps).getScriptPathFromMetainfo(anyString()); Metainfo metainfo = new Metainfo(); Application application = new Application(); metainfo.setApplication(application); doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString()); doReturn(metainfo).when(mockAps).getMetainfo(); try {// www .ja va 2 s . c o m doReturn(true).when(mockAps).isMaster(anyString()); doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"), eq("mockcontainer_1"), any(HeartBeatResponse.class), eq("scripts/hbase_master.py"), eq(600L)); doReturn(conf).when(mockAps).getConfig(); } catch (SliderException e) { } doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(), anyString(), anyMap()); expect(access.isApplicationLive()).andReturn(true).anyTimes(); ClusterDescription desc = new ClusterDescription(); desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181"); desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE"); expect(access.getClusterStatus()).andReturn(desc).anyTimes(); AggregateConf aggConf = new AggregateConf(); ConfTreeOperations treeOps = aggConf.getAppConfOperations(); treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0"); treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); treeOps.set("java_home", "/usr/jdk7/"); treeOps.set("site.fs.defaultFS", "hdfs://c6409.ambari.apache.org:8020"); treeOps.set(InternalKeys.INTERNAL_DATA_DIR_PATH, "hdfs://c6409.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/data"); expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes(); replay(access, ctx, container, sliderFileSystem); List<Container> containers = new ArrayList<Container>(); containers.add(container); Map<Integer, ProviderRole> providerRoleMap = new HashMap<Integer, ProviderRole>(); ProviderRole providerRole = new ProviderRole(role, 1); providerRoleMap.put(1, providerRole); mockAps.rebuildContainerDetails(containers, "mockcontainer_1", providerRoleMap); return mockAps; }