List of usage examples for org.apache.hadoop.yarn.api.protocolrecords AllocateResponse getNumClusterNodes
@Public @Stable public abstract int getNumClusterNodes();
From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java
License:Apache License
@Override public AllocateResponse allocate(float progressIndicator) throws YarnRemoteException { AllocateResponse allocateResponse = null; ArrayList<ResourceRequest> askList = null; ArrayList<ContainerId> releaseList = null; AllocateRequest allocateRequest = null; try {// w ww. j a v a 2s . c o m synchronized (this) { askList = new ArrayList<ResourceRequest>(ask); releaseList = new ArrayList<ContainerId>(release); // optimistically clear this collection assuming no RPC failure ask.clear(); release.clear(); allocateRequest = BuilderUtils.newAllocateRequest(appAttemptId, lastResponseId, progressIndicator, askList, releaseList); } allocateResponse = rmClient.allocate(allocateRequest); AMResponse response = allocateResponse.getAMResponse(); synchronized (this) { // update these on successful RPC clusterNodeCount = allocateResponse.getNumClusterNodes(); lastResponseId = response.getResponseId(); clusterAvailableResources = response.getAvailableResources(); } } finally { // TODO how to differentiate remote yarn exception vs error in rpc if (allocateResponse == null) { // we hit an exception in allocate() // preserve ask and release for next call to allocate() synchronized (this) { release.addAll(releaseList); // requests could have been added or deleted during call to allocate // If requests were added/removed then there is nothing to do since // the ResourceRequest object in ask would have the actual new value. // If ask does not have this ResourceRequest then it was unchanged and // so we can add the value back safely. // This assumes that there will no concurrent calls to allocate() and // so we dont have to worry about ask being changed in the // synchronized block at the beginning of this method. for (ResourceRequest oldAsk : askList) { if (!ask.contains(oldAsk)) { ask.add(oldAsk); } } } } } return allocateResponse; }
From source file:disAMS.AMRMClient.Impl.AMRMClientImpl.java
License:Apache License
@Override public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException { Preconditions.checkArgument(progressIndicator >= 0, "Progress indicator should not be negative"); AllocateResponse allocateResponse = null; List<ResourceRequest> askList = null; List<ContainerId> releaseList = null; AllocateRequest allocateRequest = null; List<String> blacklistToAdd = new ArrayList<String>(); List<String> blacklistToRemove = new ArrayList<String>(); try {/*from ww w . jav a 2 s.c o m*/ synchronized (this) { askList = new ArrayList<ResourceRequest>(ask.size()); for (ResourceRequest r : ask) { // create a copy of ResourceRequest as we might change it while the // RPC layer is using it to send info across askList.add(ResourceRequest.newInstance(r.getPriority(), r.getResourceName(), r.getCapability(), r.getNumContainers(), r.getRelaxLocality(), r.getNodeLabelExpression())); } releaseList = new ArrayList<ContainerId>(release); // optimistically clear this collection assuming no RPC failure ask.clear(); release.clear(); blacklistToAdd.addAll(blacklistAdditions); blacklistToRemove.addAll(blacklistRemovals); ResourceBlacklistRequest blacklistRequest = (blacklistToAdd != null) || (blacklistToRemove != null) ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove) : null; allocateRequest = AllocateRequest.newInstance(lastResponseId, progressIndicator, askList, releaseList, blacklistRequest); // clear blacklistAdditions and blacklistRemovals before // unsynchronized part blacklistAdditions.clear(); blacklistRemovals.clear(); } try { allocateResponse = rmClient.allocate(allocateRequest); } catch (ApplicationMasterNotRegisteredException e) { LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing."); synchronized (this) { release.addAll(this.pendingRelease); blacklistAdditions.addAll(this.blacklistedNodes); for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr : remoteRequestsTable.values()) { for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) { for (ResourceRequestInfo request : capabalities.values()) { addResourceRequestToAsk(request.remoteRequest); } } } } // re register with RM registerApplicationMaster(); allocateResponse = allocate(progressIndicator); return allocateResponse; } synchronized (this) { // update these on successful RPC clusterNodeCount = allocateResponse.getNumClusterNodes(); lastResponseId = allocateResponse.getResponseId(); clusterAvailableResources = allocateResponse.getAvailableResources(); if (!allocateResponse.getNMTokens().isEmpty()) { populateNMTokens(allocateResponse.getNMTokens()); } if (allocateResponse.getAMRMToken() != null) { updateAMRMToken(allocateResponse.getAMRMToken()); } if (!pendingRelease.isEmpty() && !allocateResponse.getCompletedContainersStatuses().isEmpty()) { removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses()); } } } finally { // TODO how to differentiate remote yarn exception vs error in rpc if (allocateResponse == null) { // we hit an exception in allocate() // preserve ask and release for next call to allocate() synchronized (this) { release.addAll(releaseList); // requests could have been added or deleted during call to allocate // If requests were added/removed then there is nothing to do since // the ResourceRequest object in ask would have the actual new value. // If ask does not have this ResourceRequest then it was unchanged and // so we can add the value back safely. // This assumes that there will no concurrent calls to allocate() and // so we dont have to worry about ask being changed in the // synchronized block at the beginning of this method. for (ResourceRequest oldAsk : askList) { if (!ask.contains(oldAsk)) { ask.add(oldAsk); } } blacklistAdditions.addAll(blacklistToAdd); blacklistRemovals.addAll(blacklistToRemove); } } } return allocateResponse; }
From source file:org.apache.tajo.master.rm.RMContainerAllocator.java
License:Apache License
public void start() { super.start(); RegisterApplicationMasterResponse response; try {/* w w w . jav a2 s . c o m*/ response = registerApplicationMaster("locahost", 10080, "http://localhost:1234"); context.setMaxContainerCapability(response.getMaximumResourceCapability().getMemory()); context.setMinContainerCapability(response.getMinimumResourceCapability().getMemory()); // If the number of cluster nodes is ZERO, it waits for available nodes. AllocateResponse allocateResponse = allocate(0.0f); while (allocateResponse.getNumClusterNodes() < 1) { try { Thread.sleep(WAIT_INTERVAL_AVAILABLE_NODES); LOG.info("Waiting for Available Cluster Nodes"); allocateResponse = allocate(0); } catch (InterruptedException e) { LOG.error(e); } } context.setNumClusterNodes(allocateResponse.getNumClusterNodes()); } catch (YarnRemoteException e) { LOG.error(e); } startAllocatorThread(); }
From source file:org.apache.tajo.master.rm.RMContainerAllocator.java
License:Apache License
public void heartbeat() throws Exception { AllocateResponse allocateResponse = allocate(context.getProgress()); AMResponse response = allocateResponse.getAMResponse(); List<Container> allocatedContainers = response.getAllocatedContainers(); LOG.info("Available Cluster Nodes: " + allocateResponse.getNumClusterNodes()); LOG.info("Available Resource: " + response.getAvailableResources()); LOG.info("Num of Allocated Containers: " + response.getAllocatedContainers().size()); if (response.getAllocatedContainers().size() > 0) { LOG.info("================================================================"); for (Container container : response.getAllocatedContainers()) { LOG.info("> Container Id: " + container.getId()); LOG.info("> Node Id: " + container.getNodeId()); LOG.info("> Resource (Mem): " + container.getResource().getMemory()); LOG.info("> State : " + container.getState()); LOG.info("> Priority: " + container.getPriority()); }/*from ww w . j a va 2s . co m*/ LOG.info("================================================================"); } Map<SubQueryId, List<Container>> allocated = new HashMap<SubQueryId, List<Container>>(); if (allocatedContainers.size() > 0) { for (Container container : allocatedContainers) { SubQueryId subQueryId = subQueryMap.get(container.getPriority()); SubQueryState state = context.getSubQuery(subQueryId).getState(); if (!(isRunningState(state) && subQueryMap.containsKey(container.getPriority()))) { releaseAssignedContainer(container.getId()); synchronized (subQueryMap) { subQueryMap.remove(container.getPriority()); } } else { if (allocated.containsKey(subQueryId)) { allocated.get(subQueryId).add(container); } else { allocated.put(subQueryId, Lists.newArrayList(container)); } } } for (Entry<SubQueryId, List<Container>> entry : allocated.entrySet()) { eventHandler.handle(new SubQueryContainerAllocationEvent(entry.getKey(), entry.getValue())); } } }
From source file:org.apache.tajo.master.rm.YarnRMContainerAllocator.java
License:Apache License
public void start() { super.start(); RegisterApplicationMasterResponse response; try {//from w w w. j a va 2 s.co m response = registerApplicationMaster("localhost", 10080, "http://localhost:1234"); // If the number of cluster nodes is ZERO, it waits for available nodes. AllocateResponse allocateResponse = allocate(0.0f); while (allocateResponse.getNumClusterNodes() < 1) { try { Thread.sleep(WAIT_INTERVAL_AVAILABLE_NODES); LOG.info("Waiting for Available Cluster Nodes"); allocateResponse = allocate(0); } catch (InterruptedException e) { LOG.error(e); } } context.getQueryMasterContext().getWorkerContext() .setNumClusterNodes(allocateResponse.getNumClusterNodes()); } catch (IOException e) { LOG.error(e); } catch (YarnException e) { LOG.error(e); } startAllocatorThread(); }
From source file:org.apache.tajo.master.rm.YarnRMContainerAllocator.java
License:Apache License
public void heartbeat() throws Exception { AllocateResponse allocateResponse = allocate(context.getProgress()); List<Container> allocatedContainers = allocateResponse.getAllocatedContainers(); long currentTime = System.currentTimeMillis(); if ((currentTime - prevReportTime.longValue()) >= reportInterval) { LOG.debug("Available Cluster Nodes: " + allocateResponse.getNumClusterNodes()); LOG.debug("Num of Allocated Containers: " + allocatedContainers.size()); LOG.info("Available Resource: " + allocateResponse.getAvailableResources()); prevReportTime.set(currentTime); }//from ww w.ja va 2 s .c om if (allocatedContainers.size() > 0) { LOG.info("================================================================"); for (Container container : allocateResponse.getAllocatedContainers()) { LOG.info("> Container Id: " + container.getId()); LOG.info("> Node Id: " + container.getNodeId()); LOG.info("> Resource (Mem): " + container.getResource().getMemory()); LOG.info("> Priority: " + container.getPriority()); } LOG.info("================================================================"); Map<ExecutionBlockId, List<Container>> allocated = new HashMap<ExecutionBlockId, List<Container>>(); for (Container container : allocatedContainers) { ExecutionBlockId executionBlockId = subQueryMap.get(container.getPriority()); SubQueryState state = context.getSubQuery(executionBlockId).getState(); if (!(SubQuery.isRunningState(state))) { releaseAssignedContainer(container.getId()); } else { if (allocated.containsKey(executionBlockId)) { allocated.get(executionBlockId).add(container); } else { allocated.put(executionBlockId, Lists.newArrayList(container)); } } } for (Entry<ExecutionBlockId, List<Container>> entry : allocated.entrySet()) { eventHandler.handle(new SubQueryContainerAllocationEvent(entry.getKey(), entry.getValue())); } } }
From source file:org.apache.twill.internal.yarn.ports.AMRMClientImpl.java
License:Apache License
@Override public AllocationResponse allocate(float progressIndicator) throws YarnRemoteException { AllocateResponse allocateResponse = null; ArrayList<ResourceRequest> askList = null; ArrayList<ContainerId> releaseList = null; AllocateRequest allocateRequest = null; try {//from w w w. java 2 s.c o m synchronized (this) { askList = new ArrayList<ResourceRequest>(ask); releaseList = new ArrayList<ContainerId>(release); // optimistically clear this collection assuming no RPC failure ask.clear(); release.clear(); allocateRequest = BuilderUtils.newAllocateRequest(appAttemptId, lastResponseId, progressIndicator, askList, releaseList); } allocateResponse = rmClient.allocate(allocateRequest); AllocationResponse response = AllocationResponses.create(allocateResponse); synchronized (this) { // update these on successful RPC clusterNodeCount = allocateResponse.getNumClusterNodes(); lastResponseId = response.getResponseId(); clusterAvailableResources = response.getAvailableResources(); } return response; } finally { // TODO how to differentiate remote YARN exception vs error in RPC if (allocateResponse == null) { // We hit an exception in allocate() // Preserve ask and release for next call to allocate() synchronized (this) { release.addAll(releaseList); // Requests could have been added or deleted during call to allocate. // If requests were added/removed then there is nothing to do since // the ResourceRequest object in ask would have the actual new value. // If ask does not have this ResourceRequest then it was unchanged and // so we can add the value back safely. // This assumes that there will no concurrent calls to allocate() and // so we don't have to worry about ask being changed in the // synchronized block at the beginning of this method. for (ResourceRequest oldAsk : askList) { if (!ask.contains(oldAsk)) { ask.add(oldAsk); } } } } } }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ResourceManagerHandler.java
License:Apache License
/** * Changed the return type to AllocateResponse which use to hold a reference to * AMResponse. //from ww w . j a v a2 s. c om * * AMResponse seems to have disappeared in CDH 4.6 * * @param requestedContainers * @param releasedContainers * @return * @throws YarnRemoteException */ public AllocateResponse allocateRequest(List<ResourceRequest> requestedContainers, List<ContainerId> releasedContainers) throws YarnRemoteException { if (amResourceManager == null) throw new IllegalStateException( "Cannot send allocation request before connecting to the resource manager!"); LOG.info("Sending allocation request" + ", requestedSize=" + requestedContainers.size() + ", releasedSize=" + releasedContainers.size()); for (ResourceRequest req : requestedContainers) LOG.info("Requesting container, host=" + req.getHostName() + ", amount=" + req.getNumContainers() + ", memory=" + req.getCapability().getMemory() + ", priority=" + req.getPriority().getPriority()); for (ContainerId rel : releasedContainers) LOG.info("Releasing container: " + rel.getId()); AllocateRequest request = Records.newRecord(AllocateRequest.class); request.setResponseId(rmRequestId.incrementAndGet()); request.setApplicationAttemptId(appAttemptId); request.addAllAsks(requestedContainers); request.addAllReleases(releasedContainers); AllocateResponse response = amResourceManager.allocate(request); //response.getAllocatedContainers() LOG.debug("Got an allocation response, " + ", responseId=" + response.getResponseId() + ", numClusterNodes=" + response.getNumClusterNodes() + ", headroom=" + response.getAvailableResources().getMemory() + ", allocatedSize=" + response.getAllocatedContainers().size() + ", updatedNodes=" + response.getUpdatedNodes().size() + ", reboot=" + response.getReboot() + ", completedSize=" + response.getCompletedContainersStatuses().size()); return response; }