List of usage examples for org.apache.hadoop.yarn.api.records ResourceRequest getPriority
@Public @Stable public abstract Priority getPriority();
Priority of the request. From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java
License:Apache License
@Override public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException { Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative"); AllocateResponse allocateResponse = null; List<ResourceRequest> askList = null; List<ContainerId> releaseList = null; AllocateRequest allocateRequest = null; List<String> blacklistToAdd = new ArrayList<String>(); List<String> blacklistToRemove = new ArrayList<String>(); try {//from ww w . ja va2s .c o m synchronized (this) { askList = new ArrayList<ResourceRequest>(ask.size()); for (ResourceRequest r : ask) { // create a copy of ResourceRequest as we might change it while the // RPC layer is using it to send info across askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(), r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression())); } releaseList = new ArrayList<ContainerId>(release); // optimistically clear this collection assuming no RPC failure ask.clear(); release.clear(); blacklistToAdd.addAll(blacklistAdditions); blacklistToRemove.addAll(blacklistRemovals); ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null) ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove) : null; allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList, releaseList, blacklistRequest); // clear blacklistAdditions and blacklistRemovals before // unsynchronized part blacklistAdditions.clear(); blacklistRemovals.clear(); } try { allocateResponse = rmClient.allocate(allocateRequest); } catch (ApplicationMasterNotRegisteredException e) { LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."); synchronized (this) { release.addAll(this.pendingRelease); blacklistAdditions.addAll(this.blacklistedNodes); for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) { for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) { for (ResourceRequestInfo request : capabalities.values()) { addResourceRequestToAsk(request.remoteRequest); } } } } // re register with RM registerApplicationMaster(); allocateResponse = allocate(progressIndicator); return allocateResponse; } synchronized (this) { // update these on successful RPC clusterNodeCount = allocateResponse.getNumClusterNodes(); lastResponseId = allocateResponse.getResponseId(); clusterAvailableResources = allocateResponse.getAvailableResources(); if (!allocateResponse.getNMTokens().isEmpty()) { populateNMTokens(allocateResponse.getNMTokens()); } if (allocateResponse.getAMRMToken() != null) { updateAMRMToken(allocateResponse.getAMRMToken()); } if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) { removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses()); } } } finally { // TODO how to differentiate remote yarn exception vs error in rpc if (allocateResponse == null) { // we hit an exception in allocate() // preserve ask and release for next call to allocate() synchronized (this) { release.addAll(releaseList); // requests could have been added or deleted during call to allocate // If requests were added/removed then there is nothing to do since // the ResourceRequest object in ask would have the actual new value. // If ask does not have this ResourceRequest then it was unchanged and // so we can add the value back safely. // This assumes that there will no concurrent calls to allocate() and // so we dont have to worry about ask being changed in the // synchronized block at the beginning of this method. for (ResourceRequest oldAsk : askList) { if (!ask.contains(oldAsk)) { ask.add(oldAsk); } } blacklistAdditions.addAll(blacklistToAdd); blacklistRemovals.addAll(blacklistToRemove); } } } return allocateResponse; }
From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java
License:Apache License
private void setupAsk(AbstractProcess proc) { ContainerSpecification cSpec = proc.getContainerSpecification(); ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class); rsrcRequest.setHostName(cSpec.getHostname()); Priority pri = Records.newRecord(Priority.class); pri.setPriority(0);//from ww w.j a v a2s . com rsrcRequest.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(cSpec.getMemory()); rsrcRequest.setCapability(capability); rsrcRequest.setNumContainers(1); AskRecord ar = new AskRecord(); ar.req = rsrcRequest; ar.proc = proc; Set<AskRecord> arSet = resource2AskMap.get(capability); if (arSet == null) { arSet = new HashSet<AskRecord>(); resource2AskMap.put(capability, arSet); } arSet.add(ar); proc2AskMap.put(proc, ar); System.err.println(proc + " -> [" + rsrcRequest.getHostName() + ", " + rsrcRequest.getNumContainers() + ", " + rsrcRequest.getPriority() + ", " + rsrcRequest.getCapability().getMemory() + "]"); asks.add(rsrcRequest); }
From source file:org.deeplearning4j.iterativereduce.runtime.Utils.java
License:Apache License
public static ResourceRequest createResourceRequest(String host, int amount, int memory) { ResourceRequest rsrcRequest = Records.newRecord(ResourceRequest.class); rsrcRequest.setHostName(host);// www. ja v a 2 s. c o m Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); rsrcRequest.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); rsrcRequest.setCapability(capability); rsrcRequest.setNumContainers(amount); LOG.debug("Created a resource request" + ", host=" + rsrcRequest.getHostName() + ", memory=" + rsrcRequest.getCapability().getMemory() + ", amount=" + rsrcRequest.getNumContainers() + ", priority=" + rsrcRequest.getPriority().getPriority()); return rsrcRequest; }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java
License:Apache License
/** * Changed the return type to AllocateResponse which use to hold a reference to * AMResponse. /* ww w.ja v a 2 s.com*/ * * AMResponse seems to have disappeared in CDH 4.6 * * @param requestedContainers * @param releasedContainers * @return * @throws YarnRemoteException */ public AllocateResponse allocateRequest(List<ResourceRequest> requestedContainers, List<ContainerId> releasedContainers) throws YarnRemoteException { if (amResourceManager == null) throw new IllegalStateException( "Cannot send allocation request before connecting to the resource manager!"); LOG.info("Sending allocation request" + ", requestedSize=" + requestedContainers.size() + ", releasedSize=" + releasedContainers.size()); for (ResourceRequest req : requestedContainers) LOG.info("Requesting container, host=" + req.getHostName() + ", amount=" + req.getNumContainers() + ", memory=" + req.getCapability().getMemory() + ", priority=" + req.getPriority().getPriority()); for (ContainerId rel : releasedContainers) LOG.info("Releasing container: " + rel.getId()); AllocateRequest request = Records.newRecord(AllocateRequest.class); request.setResponseId(rmRequestId.incrementAndGet()); request.setApplicationAttemptId(appAttemptId); request.addAllAsks(requestedContainers); request.addAllReleases(releasedContainers); AllocateResponse response = amResourceManager.allocate(request); //response.getAllocatedContainers() LOG.debug("Got an allocation response, " + ", responseId=" + response.getResponseId() + ", numClusterNodes=" + response.getNumClusterNodes() + ", headroom=" + response.getAvailableResources().getMemory() + ", allocatedSize=" + response.getAllocatedContainers().size() + ", updatedNodes=" + response.getUpdatedNodes().size() + ", reboot=" + response.getReboot() + ", completedSize=" + response.getCompletedContainersStatuses().size()); return response; }
From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java
License:Apache License
@Test public void testLegacyAnyCount() throws Exception { DefaultContainerAllocator allocator = new DefaultContainerAllocator(); allocator.setConfiguration(new Configuration()); allocator.setLocality(false);// w w w .j a v a2 s . co m TestUtils.callMethod("internalInit", allocator); allocator.allocateContainers(2); List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator); assertThat(createRequests, notNullValue()); assertThat(createRequests.size(), is(1)); ResourceRequest req0 = createRequests.get(0); assertThat(req0.getResourceName(), is("*")); assertThat(req0.getPriority().getPriority(), is(0)); assertThat(req0.getNumContainers(), is(2)); assertThat(req0.getRelaxLocality(), is(true)); }
From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java
License:Apache License
@Test public void testAnyHostNoLocalityRequests() throws Exception { DefaultContainerAllocator allocator = new DefaultContainerAllocator(); allocator.setConfiguration(new Configuration()); allocator.setLocality(false);// w w w. java 2s .c om TestUtils.callMethod("internalInit", allocator); ContainerAllocateData data = new ContainerAllocateData(); data.addAny(2); allocator.allocateContainers(data); List<ResourceRequest> createRequests = TestUtils.callMethod("createRequests", allocator); assertThat(createRequests, notNullValue()); assertThat(createRequests.size(), is(1)); ResourceRequest req0 = createRequests.get(0); assertThat(req0.getResourceName(), is("*")); assertThat(req0.getPriority().getPriority(), is(0)); assertThat(req0.getNumContainers(), is(2)); assertThat(req0.getRelaxLocality(), is(true)); }
From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java
License:Apache License
private static void assertResourceRequest(ResourceRequest req, String resourceName, Integer priority, Integer numContainers, Boolean relaxLocality, Integer capMemory, Integer capCores) { if (resourceName != null) { assertThat(req.getResourceName(), is(resourceName)); }// ww w . j av a 2 s . c o m if (priority != null) { assertThat(req.getPriority().getPriority(), is(priority)); } if (numContainers != null) { assertThat(req.getNumContainers(), is(numContainers)); } if (relaxLocality != null) { assertThat(req.getRelaxLocality(), is(relaxLocality)); } if (capMemory != null) { assertThat(req.getCapability().getMemory(), is(capMemory)); } if (capCores != null) { assertThat(req.getCapability().getVirtualCores(), is(capCores)); } }
From source file:org.springframework.yarn.am.allocate.DefaultContainerAllocatorTests.java
License:Apache License
private static ResourceRequest matchRequest(List<ResourceRequest> createRequests, String name, int priority, int containers, boolean relax) { for (ResourceRequest r : createRequests) { if (r.getResourceName().equals(name) && r.getPriority().getPriority() == priority && r.getNumContainers() == containers && r.getRelaxLocality() == relax) { return r; }/*from w w w . j av a 2 s. c o m*/ } return null; }