List of usage examples for org.apache.hadoop.yarn.api.records Container getNodeId
@Public @Stable public abstract NodeId getNodeId();
From source file:de.huberlin.wbi.hiway.scheduler.gq.GreedyQueue.java
License:Apache License
@Override public TaskInstance scheduleTaskToContainer(Container container) { numberOfRemainingTasks--;//from w w w . ja v a 2 s . c o m numberOfRunningTasks++; TaskInstance task = queue.remove(); /* log */ WorkflowDriver.Logger.writeToStdout("GQ Assigned task " + task + " to container " + container.getId() + "@" + container.getNodeId().getHost()); task.incTries(); return task; }
From source file:de.huberlin.wbi.hiway.scheduler.ma.MemoryAware.java
License:Apache License
@Override public TaskInstance scheduleTaskToContainer(Container container) { numberOfRemainingTasks--;/*from w ww. ja va 2s. c om*/ numberOfRunningTasks++; int memory = container.getResource().getMemory(); Queue<TaskInstance> queue = queuePerMem.get(memory); TaskInstance task = queue.remove(); WorkflowDriver.Logger.writeToStdout("MA Assigned task " + task + " to container " + container.getId() + "@" + container.getNodeId().getHost() + ":" + container.getResource().getVirtualCores() + ":" + container.getResource().getMemory()); task.incTries(); return task; }
From source file:de.huberlin.wbi.hiway.scheduler.StaticScheduler.java
License:Apache License
@Override public TaskInstance scheduleTaskToContainer(Container container) { numberOfRemainingTasks--;//w w w. j a v a 2 s . c o m numberOfRunningTasks++; String node = container.getNodeId().getHost(); if (HiWayConfiguration.verbose) WorkflowDriver.Logger.writeToStdout("Looking for task on container " + container.getId() + " on node " + node + "; Queue:" + readyTasksByNode.get(node).toString()); TaskInstance task = readyTasksByNode.get(node).remove(); WorkflowDriver.Logger .writeToStdout("SS Assigned task " + task + " to container " + container.getId() + "@" + node); task.incTries(); return task; }
From source file:de.huberlin.wbi.hiway.scheduler.TestScheduler.java
License:Apache License
private static void run(Scheduler scheduler, List<String> nodeNames, List<String> taskNames, int[][] runtimes) { Queue<NodeId> availableNodes = new LinkedList<>(); for (String nodeName : nodeNames) { NodeId node = NodeId.newInstance(nodeName, 0); availableNodes.add(node);//from www.ja v a 2 s . co m } Map<Container, TaskInstance> runningTasks = new HashMap<>(); Map<Container, Long> finishTimes = new HashMap<>(); int runningId = 0; long clock = 0; while (!scheduler.nothingToSchedule() || !runningTasks.isEmpty()) { if (!scheduler.nothingToSchedule() && !availableNodes.isEmpty()) { NodeId nodeId = availableNodes.remove(); ContainerId containerId = ContainerId.newContainerId(null, runningId++); Container container = Container.newInstance(containerId, nodeId, "", null, null, null); TaskInstance task = scheduler.getNextTask(container); runningTasks.put(container, task); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; finishTimes.put(container, clock + runtime); } for (Container container : finishTimes.keySet()) { if (clock == finishTimes.get(container)) { NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = ContainerStatus.newInstance(container.getId(), null, "", 0); TaskInstance task = runningTasks.get(container); task.setCompleted(); long runtime = (runtimes == null) ? 1 : runtimes[nodeNames.indexOf(nodeId.getHost())][taskNames.indexOf(task.getTaskName())]; scheduler.taskCompleted(task, containerStatus, runtime); runningTasks.remove(container); availableNodes.add(nodeId); } } clock++; } }
From source file:edu.cmu.graphchi.toolkits.collaborative_filtering.yarn.ApplicationMaster.java
License:Apache License
public void startContainer(Container c, RecommenderPool pool) { LOG.info(//w ww .ja v a2 s . c om "Launching Rec Pool command on a new container." + ", containerId=" + c.getId() + ", containerNode=" + c.getNodeId().getHost() + ":" + c.getNodeId().getPort() + ", containerNodeURI=" + c.getNodeHttpAddress() + ", containerResourceMemory" + c.getResource().getMemory()); LaunchContainerRunnable runnableLaunchContainer = new LaunchContainerRunnable(c, containerListener, pool); Thread launchThread = new Thread(runnableLaunchContainer); // launch and start the container on a separate thread to keep // the main thread unblocked // as all containers may not be allocated at one go. launchThreads.add(launchThread); launchThread.start(); }
From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * Determines whether or not a container is the one on which the CC should reside * /*from w w w. ja v a2s .c om*/ * @param c * The container in question * @return True if the container should have the CC process on it, false otherwise. */ boolean containerIsCC(Container c) { String containerHost = c.getNodeId().getHost(); try { InetAddress containerIp = InetAddress.getByName(containerHost); LOG.info(containerIp.getCanonicalHostName()); InetAddress ccIp = InetAddress.getByName(cC.getClusterIp()); LOG.info(ccIp.getCanonicalHostName()); return containerIp.getCanonicalHostName().equals(ccIp.getCanonicalHostName()); } catch (UnknownHostException e) { return false; } }
From source file:edu.uci.ics.asterix.aoya.AsterixApplicationMaster.java
License:Apache License
/** * Attempts to find the Node in the Cluster Description that matches this container * //from w w w .j a v a 2 s . c o m * @param c * The container to resolve * @return The node this container corresponds to * @throws java.net.UnknownHostException * if the container isn't present in the description */ Node containerToNode(Container c, Cluster cl) throws UnknownHostException { String containerHost = c.getNodeId().getHost(); InetAddress containerIp = InetAddress.getByName(containerHost); LOG.info("Resolved Container IP: " + containerIp); for (Node node : cl.getNode()) { InetAddress nodeIp = InetAddress.getByName(node.getClusterIp()); LOG.info(nodeIp + "?=" + containerIp); if (nodeIp.equals(containerIp)) return node; } //if we find nothing, this is bad... throw new java.net.UnknownHostException("Could not resolve container" + containerHost + " to node"); }
From source file:edu.uci.ics.hyracks.yarn.am.HyracksYarnApplicationMaster.java
License:Apache License
private synchronized void processAllocation(List<Container> allocatedContainers, List<ContainerStatus> completedContainers) { System.err.println(allocatedContainers); for (Container c : allocatedContainers) { System.err.println("Got container: " + c.getContainerStatus()); NodeId nodeId = c.getNodeId(); Resource resource = c.getResource(); Set<AskRecord> arSet = resource2AskMap.get(resource); boolean found = false; if (arSet != null) { AskRecord wildcardMatch = null; AskRecord nameMatch = null;// w w w . j ava2 s . c o m for (AskRecord ar : arSet) { ResourceRequest req = ar.req; if (ar.allocation == null) { if ("*".equals(req.getHostName()) && wildcardMatch == null) { wildcardMatch = ar; } if (req.getHostName().equals(nodeId.getHost()) && nameMatch == null) { nameMatch = ar; break; } } } if (nameMatch != null) { found = true; nameMatch.allocation = c; } else if (wildcardMatch != null) { found = true; wildcardMatch.allocation = c; } } if (!found) { System.err.println("Unknown request satisfied: " + resource); } } }
From source file:eu.stratosphere.yarn.ApplicationMaster.java
License:Apache License
private void run() throws Exception { //Utils.logFilesInCurrentDirectory(LOG); // Initialize clients to ResourceManager and NodeManagers Configuration conf = Utils.initializeYarnConfiguration(); FileSystem fs = FileSystem.get(conf); Map<String, String> envs = System.getenv(); final String currDir = envs.get(Environment.PWD.key()); final String logDirs = envs.get(Environment.LOG_DIRS.key()); final String ownHostname = envs.get(Environment.NM_HOST.key()); final String appId = envs.get(Client.ENV_APP_ID); final String clientHomeDir = envs.get(Client.ENV_CLIENT_HOME_DIR); final String applicationMasterHost = envs.get(Environment.NM_HOST.key()); final String remoteStratosphereJarPath = envs.get(Client.STRATOSPHERE_JAR_PATH); final String shipListString = envs.get(Client.ENV_CLIENT_SHIP_FILES); final String yarnClientUsername = envs.get(Client.ENV_CLIENT_USERNAME); final int taskManagerCount = Integer.valueOf(envs.get(Client.ENV_TM_COUNT)); final int memoryPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_MEMORY)); final int coresPerTaskManager = Integer.valueOf(envs.get(Client.ENV_TM_CORES)); int heapLimit = Utils.calculateHeapSize(memoryPerTaskManager); if (currDir == null) { throw new RuntimeException("Current directory unknown"); }//from w ww . ja va 2 s .co m if (ownHostname == null) { throw new RuntimeException("Own hostname (" + Environment.NM_HOST + ") not set."); } LOG.info("Working directory " + currDir); // load Stratosphere configuration. Utils.getStratosphereConfiguration(currDir); final String localWebInterfaceDir = currDir + "/resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME; // Update yaml conf -> set jobManager address to this machine's address. FileInputStream fis = new FileInputStream(currDir + "/stratosphere-conf.yaml"); BufferedReader br = new BufferedReader(new InputStreamReader(fis)); Writer output = new BufferedWriter(new FileWriter(currDir + "/stratosphere-conf-modified.yaml")); String line; while ((line = br.readLine()) != null) { if (line.contains(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)) { output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n"); } else if (line.contains(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY)) { output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + "\n"); } else { output.append(line + "\n"); } } // just to make sure. output.append(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY + ": " + ownHostname + "\n"); output.append(ConfigConstants.JOB_MANAGER_WEB_ROOT_PATH_KEY + ": " + localWebInterfaceDir + "\n"); output.append(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY + ": " + logDirs + "\n"); output.close(); br.close(); File newConf = new File(currDir + "/stratosphere-conf-modified.yaml"); if (!newConf.exists()) { LOG.warn("modified yaml does not exist!"); } Utils.copyJarContents("resources/" + ConfigConstants.DEFAULT_JOB_MANAGER_WEB_PATH_NAME, ApplicationMaster.class.getProtectionDomain().getCodeSource().getLocation().getPath()); JobManager jm; { String pathToNepheleConfig = currDir + "/stratosphere-conf-modified.yaml"; String[] args = { "-executionMode", "cluster", "-configDir", pathToNepheleConfig }; // start the job manager jm = JobManager.initialize(args); // Start info server for jobmanager jm.startInfoServer(); } AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager LOG.info("registering ApplicationMaster"); rmClient.registerApplicationMaster(applicationMasterHost, 0, "http://" + applicationMasterHost + ":" + GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, "undefined")); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memoryPerTaskManager); capability.setVirtualCores(coresPerTaskManager); // Make container requests to ResourceManager for (int i = 0; i < taskManagerCount; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); LOG.info("Requesting TaskManager container " + i); rmClient.addContainerRequest(containerAsk); } LocalResource stratosphereJar = Records.newRecord(LocalResource.class); LocalResource stratosphereConf = Records.newRecord(LocalResource.class); // register Stratosphere Jar with remote HDFS final Path remoteJarPath = new Path(remoteStratosphereJarPath); Utils.registerLocalResource(fs, remoteJarPath, stratosphereJar); // register conf with local fs. Path remoteConfPath = Utils.setupLocalResource(conf, fs, appId, new Path("file://" + currDir + "/stratosphere-conf-modified.yaml"), stratosphereConf, new Path(clientHomeDir)); LOG.info("Prepared localresource for modified yaml: " + stratosphereConf); boolean hasLog4j = new File(currDir + "/log4j.properties").exists(); // prepare the files to ship LocalResource[] remoteShipRsc = null; String[] remoteShipPaths = shipListString.split(","); if (!shipListString.isEmpty()) { remoteShipRsc = new LocalResource[remoteShipPaths.length]; { // scope for i int i = 0; for (String remoteShipPathStr : remoteShipPaths) { if (remoteShipPathStr == null || remoteShipPathStr.isEmpty()) { continue; } remoteShipRsc[i] = Records.newRecord(LocalResource.class); Path remoteShipPath = new Path(remoteShipPathStr); Utils.registerLocalResource(fs, remoteShipPath, remoteShipRsc[i]); i++; } } } // respect custom JVM options in the YAML file final String javaOpts = GlobalConfiguration.getString(ConfigConstants.STRATOSPHERE_JVM_OPTIONS, ""); // Obtain allocated containers and launch int allocatedContainers = 0; int completedContainers = 0; while (allocatedContainers < taskManagerCount) { AllocateResponse response = rmClient.allocate(0); for (Container container : response.getAllocatedContainers()) { LOG.info("Got new Container for TM " + container.getId() + " on host " + container.getNodeId().getHost()); ++allocatedContainers; // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); String tmCommand = "$JAVA_HOME/bin/java -Xmx" + heapLimit + "m " + javaOpts; if (hasLog4j) { tmCommand += " -Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-log4j.log\" -Dlog4j.configuration=file:log4j.properties"; } tmCommand += " eu.stratosphere.yarn.YarnTaskManagerRunner -configDir . " + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stdout.log" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/taskmanager-stderr.log"; ctx.setCommands(Collections.singletonList(tmCommand)); LOG.info("Starting TM with command=" + tmCommand); // copy resources to the TaskManagers. Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(2); localResources.put("stratosphere.jar", stratosphereJar); localResources.put("stratosphere-conf.yaml", stratosphereConf); // add ship resources if (!shipListString.isEmpty()) { Preconditions.checkNotNull(remoteShipRsc); for (int i = 0; i < remoteShipPaths.length; i++) { localResources.put(new Path(remoteShipPaths[i]).getName(), remoteShipRsc[i]); } } ctx.setLocalResources(localResources); // Setup CLASSPATH for Container (=TaskTracker) Map<String, String> containerEnv = new HashMap<String, String>(); Utils.setupEnv(conf, containerEnv); //add stratosphere.jar to class path. containerEnv.put(Client.ENV_CLIENT_USERNAME, yarnClientUsername); ctx.setEnvironment(containerEnv); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try { Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); ctx.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } LOG.info("Launching container " + allocatedContainers); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { ++completedContainers; LOG.info("Completed container (while allocating) " + status.getContainerId() + ". Total Completed:" + completedContainers); LOG.info("Diagnostics " + status.getDiagnostics()); } Thread.sleep(100); } // Now wait for containers to complete while (completedContainers < taskManagerCount) { AllocateResponse response = rmClient.allocate(completedContainers / taskManagerCount); for (ContainerStatus status : response.getCompletedContainersStatuses()) { ++completedContainers; LOG.info("Completed container " + status.getContainerId() + ". Total Completed:" + completedContainers); LOG.info("Diagnostics " + status.getDiagnostics()); } Thread.sleep(5000); } LOG.info("Shutting down JobManager"); jm.shutdown(); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }
From source file:gobblin.yarn.YarnService.java
License:Apache License
@SuppressWarnings("unused") @Subscribe/*from w w w. j a va2 s .c o m*/ public void handleNewContainerRequest(NewContainerRequest newContainerRequest) { if (!this.maxResourceCapacity.isPresent()) { LOGGER.error(String.format( "Unable to handle new container request as maximum resource capacity is not available: " + "[memory (MBs) requested = %d, vcores requested = %d]", this.requestedContainerMemoryMbs, this.requestedContainerCores)); return; } requestContainer(newContainerRequest.getReplacedContainer().transform(new Function<Container, String>() { @Override public String apply(Container container) { return container.getNodeId().getHost(); } })); }