List of usage examples for org.apache.hadoop.yarn.api.records Container getResource
@Public @Stable public abstract Resource getResource();
Resource allocated to the container. From source file:com.inforefiner.hdata.ApplicationMaster.java
License:Apache License
private static void publishContainerStartEvent(final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId);/*from w ww . j a va 2s . co m*/ entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo("Node", container.getNodeId().toString()); event.addEventInfo("Resources", container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
From source file:com.scistor.dshell.ScistorApplicationMaster.java
License:Apache License
private static void publishContainerStartEvent(TimelineClient timelineClient, Container container) throws IOException, YarnException { TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser().toString()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo("Node", container.getNodeId().toString()); event.addEventInfo("Resources", container.getResource().toString()); entity.addEvent(event);/* w ww.j a v a 2 s . c o m*/ timelineClient.putEntities(entity); }
From source file:de.huberlin.wbi.hiway.am.benchmark.PerfectDaxGreedyQueue.java
License:Apache License
@Override public TaskInstance scheduleTaskToContainer(Container container) { numberOfRemainingTasks--;/*from w ww .j a v a2 s . c om*/ numberOfRunningTasks++; // compute and log the service time (from request to allocation) for this container logRequestServiceTime(container); // biggest containers first such that we match the biggest possible task to each container. // queue.sort((t1,t2)-> - Long.compare(t1.getPeakMemoryBytes(),t2.getPeakMemoryBytes())); // search the container in the queue with the lowest wastage (but do not change the order to avoid starvation) double optWastedBytes = container.getResource().getMemory() * 1e6; DaxTaskInstance bestfit = null; for (DaxTaskInstance task : queue) { double wastedBytes = container.getResource().getMemory() * 1e6 - task.getPeakMemoryBytes(); if (wastedBytes >= 0 /* task fits */ && wastedBytes < optWastedBytes) { optWastedBytes = wastedBytes; bestfit = task; } else { WorkflowDriver.Logger.writeToStdout(String.format( "PerfDaxGQ skipped task %s (peak_mem_bytes=%s) for %s MB container", task.getTaskName(), task.getPeakMemoryBytes(), container.getResource().getMemory())); } } if (bestfit != null) { WorkflowDriver.Logger.writeToStdout(String.format( "PerfDaxGQ Assigned task %s (peak_mem_bytes %s) to container %s@%s (memory %s MB)", bestfit, bestfit.getPeakMemoryBytes(), container.getId(), container.getNodeId().getHost(), container.getResource().getMemory())); bestfit.incTries(); } return bestfit; }
From source file:de.huberlin.wbi.hiway.am.benchmark.PerfectDaxGreedyQueue.java
License:Apache License
private void logRequestServiceTime(Container allocatedContainer) { int containerSizeMB = allocatedContainer.getResource().getMemory(); if (requestPublishedTime.containsKey(containerSizeMB)) { long responseTime = System.currentTimeMillis() - requestPublishedTime.get(containerSizeMB); requestAnswerTimes.putIfAbsent(containerSizeMB, new ArrayList<>(100)); requestAnswerTimes.get(containerSizeMB).add(responseTime); // remove the request to make sure it's only used once. requestPublishedTime.remove(containerSizeMB); WorkflowDriver.Logger.writeToStdout(String.format( "allocationWaitTime: container_size_MB=%s, wait_time_ms=%s", containerSizeMB, responseTime)); } else {/* w w w . j a v a2 s .c o m*/ WorkflowDriver.Logger.writeToStdErr(String.format( "Streak design assumption didn't hold, launching container of size %s MB for which I don't have a request in requestPublishedTime. State of requestPublishedTime: %s ", containerSizeMB, requestPublishedTime.entrySet().stream().map(entry -> entry.getKey() + ": " + entry.getValue()) .collect(Collectors.joining("; ")))); } }
From source file:de.huberlin.wbi.hiway.am.dax.DaxTaskInstance.java
License:Apache License
/** * Infers the memory limit of the docker container (see {@link #getCommand()} from the amount memory allocated to the YARN container. *//*from w w w . j ava 2 s. c o m*/ @Override public Map<String, LocalResource> buildScriptsAndSetResources(Container container) { // inform the task about its memory limits (needed to build the command) setContainerMemoryLimitBytes(container.getResource().getMemory() * 1024L * 1024L); // this calls getCommand() which requires #containerMemoryLimitBytes to be set correctly. return super.buildScriptsAndSetResources(container); }
From source file:de.huberlin.wbi.hiway.scheduler.ma.MemoryAware.java
License:Apache License
@Override public TaskInstance scheduleTaskToContainer(Container container) { numberOfRemainingTasks--;//from ww w. j av a2s . c o m numberOfRunningTasks++; int memory = container.getResource().getMemory(); Queue<TaskInstance> queue = queuePerMem.get(memory); TaskInstance task = queue.remove(); WorkflowDriver.Logger.writeToStdout("MA Assigned task " + task + " to container " + container.getId() + "@" + container.getNodeId().getHost() + ":" + container.getResource().getVirtualCores() + ":" + container.getResource().getMemory()); task.incTries(); return task; }
From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java
License:Apache License
/** * Main run function for the application master * * @throws YarnException//from w ww. java2 s.co m * @throws IOException */ @SuppressWarnings({ "unchecked" }) public boolean run() throws YarnException, IOException { yarnClient.start(); LOG.info("Starting ApplicationMaster"); Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<?> token = iter.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amRMClient = AMRMClient.createAMRMClient(); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); //TODO: Figure out how to do this. List<NodeReport> reports = this.yarnClient.getNodeReports(); LOG.info("Cluster Status"); List<Resource> availableResources = new ArrayList<Resource>(); for (NodeReport nr : reports) { LOG.info(" NodeId: " + nr.getNodeId() + " Capabilities " + nr.getCapability() + " Used Resources " + nr.getUsed()); int availableMem = nr.getCapability().getMemory() - nr.getUsed().getMemory(); int availableVCores = nr.getCapability().getVirtualCores() - nr.getUsed().getVirtualCores(); Resource resource = Resource.newInstance(availableMem, availableVCores); availableResources.add(resource); } /*Based on available resources scheduler should decide the best allocation of recommenders to resources and return a list of resources that should be requested from the ResourceManager*/ DataSetDescription datasetDesc = new DataSetDescription(this.setup.dataMetadataFile); List<RecommenderPool> recommenderPools = RecommenderScheduler.splitRecommenderPool(availableResources, recommenders, datasetDesc, this.setup.nShards); for (RecommenderPool res : recommenderPools) { ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority); LOG.info("CONTAINER ASK: " + containerAsk); amRMClient.addContainerRequest(containerAsk); } float progress = 0; List<RecommenderPool> pendingPools = new ArrayList<RecommenderPool>(); for (RecommenderPool p : recommenderPools) pendingPools.add(p); this.numTotalContainers = recommenderPools.size(); this.numCompletedContainers.set(0); this.numAllocatedContainers.set(0); while (numCompletedContainers.get() != numTotalContainers) { try { Thread.sleep(200); AllocateResponse allocResp = amRMClient.allocate(progress); List<Container> newContainers = allocResp.getAllocatedContainers(); List<ContainerStatus> completedContainers = allocResp.getCompletedContainersStatuses(); if (this.numAllocatedContainers.get() >= this.numTotalContainers && pendingPools.size() != 0) { //Ask for new containers for pending pools LOG.warn("The number of allocated containers has exceeded number of total containers, but " + "the pending pool size is still not 0. Asking for new containers from RM"); for (RecommenderPool res : pendingPools) { ContainerRequest containerAsk = setupContainerAskForRM(res.getTotalMemory(), requestPriority); LOG.info("NEW CONTAINER ASK: " + containerAsk); amRMClient.addContainerRequest(containerAsk); } } if (newContainers.size() > 0) { LOG.info("Allocated " + newContainers.size() + " new containers"); numAllocatedContainers.addAndGet(newContainers.size()); for (Container container : newContainers) { //Find matching recommender pool from pendingRecommender pools. RecommenderPool pool = null; for (RecommenderPool p : pendingPools) { if (p.getTotalMemory() == container.getResource().getMemory()) { pool = p; break; } } if (pool == null) { LOG.warn("No Takers for Container " + container + " Releasing container"); amRMClient.releaseAssignedContainer(container.getId()); } else { startContainer(container, pool); //This pool has now got a container. Remove it from pending pools pendingPools.remove(pool); } } } onContainersCompleted(completedContainers); } catch (InterruptedException ex) { } } finish(); return success; }
From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java
License:Apache License
public void startContainer(Container c, RecommenderPool pool) { LOG.info(// w ww . j a v a 2s. com "Launching Rec Pool command on a new container." + ", containerId=" + c.getId() + ", containerNode=" + c.getNodeId().getHost() + ":" + c.getNodeId().getPort() + ", containerNodeURI=" + c.getNodeHttpAddress() + ", containerResourceMemory" + c.getResource().getMemory()); LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(c, containerListener, pool); Thread launchThread = new Thread(runnableLaunchContainer); // launch and start the container on a separate thread to keep // the main thread unblocked // as all containers may not be allocated at one go. launchThreads.add(launchThread); launchThread.start(); }
From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java
License:Apache License
private synchronized void processAllocation(List<Container> allocatedContainers, List<ContainerStatus> completedContainers) { System.err.println(allocatedContainers); for (Container c : allocatedContainers) { System.err.println("Got container: " + c.getContainerStatus()); NodeId nodeId = c.getNodeId();/*from w w w . ja v a2 s .c om*/ Resource resource = c.getResource(); Set<AskRecord> arSet = resource2AskMap.get(resource); boolean found = false; if (arSet != null) { AskRecord wildcardMatch = null; AskRecord nameMatch = null; for (AskRecord ar : arSet) { ResourceRequest req = ar.req; if (ar.allocation == null) { if ("*".equals(req.getHostName()) && wildcardMatch == null) { wildcardMatch = ar; } if (req.getHostName().equals(nodeId.getHost()) && nameMatch == null) { nameMatch = ar; break; } } } if (nameMatch != null) { found = true; nameMatch.allocation = c; } else if (wildcardMatch != null) { found = true; wildcardMatch.allocation = c; } } if (!found) { System.err.println("Unknown request satisfied: " + resource); } } }
From source file:gobblin.yarn.YarnService.java
License:Apache License
private String buildContainerCommand(Container container, String helixInstanceName) { String containerProcessName = GobblinYarnTaskRunner.class.getSimpleName(); return new StringBuilder().append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java") .append(" -Xmx").append(container.getResource().getMemory()).append("M").append(" ") .append(JvmUtils.formatJvmArguments(this.containerJvmArgs)).append(" ") .append(GobblinYarnTaskRunner.class.getName()).append(" --") .append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME).append(" ") .append(this.applicationName).append(" --") .append(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME).append(" ") .append(helixInstanceName).append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR) .append(File.separator).append(containerProcessName).append(".").append(ApplicationConstants.STDOUT) .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator) .append(containerProcessName).append(".").append(ApplicationConstants.STDERR).toString(); }