List of usage examples for org.apache.hadoop.yarn.api.records Container getResource
@Public @Stable public abstract Resource getResource();
Resource allocated to the container. From source file:org.apache.reef.runtime.yarn.driver.YARNResourceLaunchHandler.java
License:Apache License
@Override public void onNext(final DriverRuntimeProtocol.ResourceLaunchProto resourceLaunchProto) { try {/*from w w w . j ava 2s . c o m*/ final String containerId = resourceLaunchProto.getIdentifier(); LOG.log(Level.FINEST, "TIME: Start ResourceLaunchProto {0}", containerId); final Container container = this.containers.get(containerId); LOG.log(Level.FINEST, "Setting up container launch container for id={0}", container.getId()); final Map<String, LocalResource> localResources = this.evaluatorSetupHelper .getResources(resourceLaunchProto); final LaunchCommandBuilder commandBuilder; switch (resourceLaunchProto.getType()) { case JVM: commandBuilder = new JavaLaunchCommandBuilder() .setClassPath(this.classpath.getEvaluatorClasspath()); break; case CLR: commandBuilder = new CLRLaunchCommandBuilder(); break; default: throw new IllegalArgumentException("Unsupported container type: " + resourceLaunchProto.getType()); } final List<String> command = commandBuilder.setErrorHandlerRID(resourceLaunchProto.getRemoteId()) .setLaunchID(resourceLaunchProto.getIdentifier()) .setConfigurationFileName(this.filenames.getEvaluatorConfigurationPath()) .setMemory((int) (this.jvmHeapFactor * container.getResource().getMemory())) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getEvaluatorStderrFileName()) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getEvaluatorStdoutFileName()) .build(); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "TIME: Run ResourceLaunchProto {0} command: `{1}` with resources: `{2}`", new Object[] { containerId, StringUtils.join(command, ' '), localResources }); } final ContainerLaunchContext ctx = YarnTypes.getContainerLaunchContext(command, localResources); this.yarnContainerManager.get().submit(container, ctx); LOG.log(Level.FINEST, "TIME: End ResourceLaunchProto {0}", containerId); } catch (final Throwable e) { LOG.log(Level.WARNING, "Error handling resource launch message: " + resourceLaunchProto, e); throw new RuntimeException(e); } }
From source file:org.apache.samza.job.yarn.refactor.YarnClusterResourceManager.java
License:Apache License
/** * Callback invoked from Yarn when containers are allocated. This translates the yarn callbacks into Samza * specific ones./*from w w w . jav a2 s .c o m*/ * @param containers the list of {@link Container} returned by Yarn. */ @Override public void onContainersAllocated(List<Container> containers) { List<SamzaResource> resources = new ArrayList<SamzaResource>(); for (Container container : containers) { log.info("Container allocated from RM on " + container.getNodeId().getHost()); final String id = container.getId().toString(); String host = container.getNodeId().getHost(); int memory = container.getResource().getMemory(); int numCores = container.getResource().getVirtualCores(); SamzaResource resource = new SamzaResource(numCores, memory, host, id); allocatedResources.put(resource, container); resources.add(resource); } _callback.onResourcesAvailable(resources); }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Callback invoked from Yarn when containers are allocated. This translates the yarn callbacks into Samza * specific ones.// ww w . jav a 2s. c om * @param containers the list of {@link Container} returned by Yarn. */ @Override public void onContainersAllocated(List<Container> containers) { List<SamzaResource> resources = new ArrayList<SamzaResource>(); for (Container container : containers) { log.info("Got allocation notification for Container ID: {} on host: {}", container.getId(), container.getNodeId().getHost()); String containerId = container.getId().toString(); String host = container.getNodeId().getHost(); int memory = container.getResource().getMemory(); int numCores = container.getResource().getVirtualCores(); SamzaResource resource = new SamzaResource(numCores, memory, host, containerId); allocatedResources.put(resource, container); resources.add(resource); } clusterManagerCallback.onResourcesAvailable(resources); }
From source file:org.apache.tajo.master.rm.RMContainerAllocator.java
License:Apache License
public void heartbeat() throws Exception { AllocateResponse allocateResponse = allocate(context.getProgress()); AMResponse response = allocateResponse.getAMResponse(); List<Container> allocatedContainers = response.getAllocatedContainers(); LOG.info("Available Cluster Nodes: " + allocateResponse.getNumClusterNodes()); LOG.info("Available Resource: " + response.getAvailableResources()); LOG.info("Num of Allocated Containers: " + response.getAllocatedContainers().size()); if (response.getAllocatedContainers().size() > 0) { LOG.info("================================================================"); for (Container container : response.getAllocatedContainers()) { LOG.info("> Container Id: " + container.getId()); LOG.info("> Node Id: " + container.getNodeId()); LOG.info("> Resource (Mem): " + container.getResource().getMemory()); LOG.info("> State : " + container.getState()); LOG.info("> Priority: " + container.getPriority()); }/*from ww w . j a v a 2 s .c o m*/ LOG.info("================================================================"); } Map<SubQueryId, List<Container>> allocated = new HashMap<SubQueryId, List<Container>>(); if (allocatedContainers.size() > 0) { for (Container container : allocatedContainers) { SubQueryId subQueryId = subQueryMap.get(container.getPriority()); SubQueryState state = context.getSubQuery(subQueryId).getState(); if (!(isRunningState(state) && subQueryMap.containsKey(container.getPriority()))) { releaseAssignedContainer(container.getId()); synchronized (subQueryMap) { subQueryMap.remove(container.getPriority()); } } else { if (allocated.containsKey(subQueryId)) { allocated.get(subQueryId).add(container); } else { allocated.put(subQueryId, Lists.newArrayList(container)); } } } for (Entry<SubQueryId, List<Container>> entry : allocated.entrySet()) { eventHandler.handle(new SubQueryContainerAllocationEvent(entry.getKey(), entry.getValue())); } } }
From source file:org.apache.tajo.master.rm.YarnRMContainerAllocator.java
License:Apache License
public void heartbeat() throws Exception { AllocateResponse allocateResponse = allocate(context.getProgress()); List<Container> allocatedContainers = allocateResponse.getAllocatedContainers(); long currentTime = System.currentTimeMillis(); if ((currentTime - prevReportTime.longValue()) >= reportInterval) { LOG.debug("Available Cluster Nodes: " + allocateResponse.getNumClusterNodes()); LOG.debug("Num of Allocated Containers: " + allocatedContainers.size()); LOG.info("Available Resource: " + allocateResponse.getAvailableResources()); prevReportTime.set(currentTime); }/* w w w . j a v a 2 s.c o m*/ if (allocatedContainers.size() > 0) { LOG.info("================================================================"); for (Container container : allocateResponse.getAllocatedContainers()) { LOG.info("> Container Id: " + container.getId()); LOG.info("> Node Id: " + container.getNodeId()); LOG.info("> Resource (Mem): " + container.getResource().getMemory()); LOG.info("> Priority: " + container.getPriority()); } LOG.info("================================================================"); Map<ExecutionBlockId, List<Container>> allocated = new HashMap<ExecutionBlockId, List<Container>>(); for (Container container : allocatedContainers) { ExecutionBlockId executionBlockId = subQueryMap.get(container.getPriority()); SubQueryState state = context.getSubQuery(executionBlockId).getState(); if (!(SubQuery.isRunningState(state))) { releaseAssignedContainer(container.getId()); } else { if (allocated.containsKey(executionBlockId)) { allocated.get(executionBlockId).add(container); } else { allocated.put(executionBlockId, Lists.newArrayList(container)); } } } for (Entry<ExecutionBlockId, List<Container>> entry : allocated.entrySet()) { eventHandler.handle(new SubQueryContainerAllocationEvent(entry.getKey(), entry.getValue())); } } }
From source file:org.apache.tez.dag.app.rm.DagAwareYarnTaskScheduler.java
License:Apache License
private synchronized List<Assignment> assignNewContainers(List<Container> newContainers, AMState appState, boolean isSession) { // try to assign the containers as node-local List<Assignment> assignments = new ArrayList<>(newContainers.size()); List<HeldContainer> unassigned = new ArrayList<>(newContainers.size()); for (Container c : newContainers) { HeldContainer hc = new HeldContainer(c); heldContainers.put(hc.getId(), hc); Resources.addTo(allocatedResources, c.getResource()); tryAssignNewContainer(hc, hc.getHost(), assignments, unassigned); }// w ww .ja va 2 s.c om // try to assign the remaining containers as rack-local List<HeldContainer> containers = unassigned; unassigned = new ArrayList<>(containers.size()); for (HeldContainer hc : containers) { tryAssignNewContainer(hc, hc.getRack(), assignments, unassigned); } // try to assign the remaining containers without locality containers = unassigned; unassigned = new ArrayList<>(containers.size()); for (HeldContainer hc : containers) { tryAssignNewContainer(hc, ResourceRequest.ANY, assignments, unassigned); } for (HeldContainer hc : unassigned) { if (reuseNewContainers) { idleTracker.add(hc); TaskRequest assigned = tryAssignReuseContainer(hc, appState, isSession); if (assigned != null) { assignments.add(new Assignment(assigned, hc.getContainer())); } } else { releaseContainer(hc); } } return assignments; }
From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
@Override public void onContainersAllocated(List<Container> containers) { if (isStopped) { return;/*from w w w. j a va 2 s .com*/ } Map<CookieContainerRequest, Container> appContainers = new HashMap<CookieContainerRequest, Container>( containers.size()); synchronized (this) { for (Container container : containers) { String location = container.getNodeId().getHost(); CookieContainerRequest assigned = getMatchingRequest(container, location); if (assigned == null) { location = RackResolver.resolve(location).getNetworkLocation(); assigned = getMatchingRequest(container, location); } if (assigned == null) { location = ResourceRequest.ANY; assigned = getMatchingRequest(container, location); } if (assigned == null) { // not matched anything. release container // Probably we cancelled a request and RM allocated that to us // before RM heard of the cancellation releaseContainer(container.getId(), null); LOG.info("No RM requests matching container: " + container); continue; } Object task = getTask(assigned); assert task != null; assignContainer(task, container, assigned); appContainers.put(assigned, container); LOG.info("Assigning container: " + container + " for task: " + task + " at locality: " + location + " resource memory: " + container.getResource().getMemory() + " cpu: " + container.getResource().getVirtualCores()); } } // upcall to app must be outside locks for (Entry<CookieContainerRequest, Container> entry : appContainers.entrySet()) { CookieContainerRequest assigned = entry.getKey(); appClient.taskAllocated(getTask(assigned), assigned.getCookie().appCookie, entry.getValue()); } }
From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
private CookieContainerRequest getMatchingRequest(Container container, String location) { Priority priority = container.getPriority(); Resource capability = container.getResource(); CookieContainerRequest assigned = null; List<? extends Collection<CookieContainerRequest>> requestsList = amRmClient.getMatchingRequests(priority, location, capability);// ww w.j a va2s . c o m if (requestsList.size() > 0) { // pick first one for (Collection<CookieContainerRequest> requests : requestsList) { Iterator<CookieContainerRequest> iterator = requests.iterator(); if (iterator.hasNext()) { assigned = requests.iterator().next(); } } } return assigned; }
From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
private void assignContainer(Object task, Container container, CookieContainerRequest assigned) { CookieContainerRequest request = removeTaskRequest(task); assert request != null; //assert assigned.equals(request); Container result = taskAllocations.put(task, container); assert result == null; containerAssigments.put(container.getId(), task); Resources.addTo(allocatedResources, container.getResource()); }
From source file:org.apache.tez.dag.app.rm.TaskScheduler.java
License:Apache License
private Container unAssignContainer(Object task, boolean releaseIfFound) { Container container = taskAllocations.remove(task); if (container == null) { return null; }/*from w w w . ja v a 2s .c o m*/ Resources.subtractFrom(allocatedResources, container.getResource()); assert allocatedResources.getMemory() >= 0; containerAssigments.remove(container.getId()); if (releaseIfFound) { releaseContainer(container.getId(), task); } return container; }