List of usage examples for org.apache.hadoop.yarn.api.records Resource setVirtualCores
@Public @Evolving public abstract void setVirtualCores(int vCores);
From source file:runtime.starter.MPJYarnClient.java
License:Open Source License
public void run() throws Exception { Map<String, String> map = System.getenv(); try {/*from ww w. j av a2 s. c om*/ mpjHomeDir = map.get("MPJ_HOME"); if (mpjHomeDir == null) { throw new Exception("[MPJRun.java]:MPJ_HOME environment found.."); } } catch (Exception exc) { System.out.println("[MPJRun.java]:" + exc.getMessage()); exc.printStackTrace(); return; } // Copy the application master jar to HDFS // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); /* Path dataset = new Path(fs.getHomeDirectory(),"/dataset"); FileStatus datasetFile = fs.getFileStatus(dataset); BlockLocation myBlocks [] = fs.getFileBlockLocations(datasetFile,0,datasetFile.getLen()); for(BlockLocation b : myBlocks){ System.out.println("\n--------------------"); System.out.println("Length "+b.getLength()); for(String host : b.getHosts()){ System.out.println("host "+host); } } */ Path source = new Path(mpjHomeDir + "/lib/mpj-app-master.jar"); String pathSuffix = hdfsFolder + "mpj-app-master.jar"; Path dest = new Path(fs.getHomeDirectory(), pathSuffix); if (debugYarn) { logger.info("Uploading mpj-app-master.jar to: " + dest.toString()); } fs.copyFromLocalFile(false, true, source, dest); FileStatus destStatus = fs.getFileStatus(dest); Path wrapperSource = new Path(mpjHomeDir + "/lib/mpj-yarn-wrapper.jar"); String wrapperSuffix = hdfsFolder + "mpj-yarn-wrapper.jar"; Path wrapperDest = new Path(fs.getHomeDirectory(), wrapperSuffix); if (debugYarn) { logger.info("Uploading mpj-yarn-wrapper.jar to: " + wrapperDest.toString()); } fs.copyFromLocalFile(false, true, wrapperSource, wrapperDest); Path userJar = new Path(jarPath); String userJarSuffix = hdfsFolder + "user-code.jar"; Path userJarDest = new Path(fs.getHomeDirectory(), userJarSuffix); if (debugYarn) { logger.info("Uploading user-code.jar to: " + userJarDest.toString()); } fs.copyFromLocalFile(false, true, userJar, userJarDest); YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); if (debugYarn) { YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); logger.info("\nNodes Information"); logger.info("Number of NM: " + metrics.getNumNodeManagers() + "\n"); List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport n : nodeReports) { logger.info("NodeId: " + n.getNodeId()); logger.info("RackName: " + n.getRackName()); logger.info("Total Memory: " + n.getCapability().getMemory()); logger.info("Used Memory: " + n.getUsed().getMemory()); logger.info("Total vCores: " + n.getCapability().getVirtualCores()); logger.info("Used vCores: " + n.getUsed().getVirtualCores() + "\n"); } } logger.info("Creating server socket at HOST " + serverName + " PORT " + serverPort + " \nWaiting for " + np + " processes to connect..."); // Creating a server socket for incoming connections try { servSock = new ServerSocket(serverPort); infoSock = new ServerSocket(); TEMP_PORT = findPort(infoSock); } catch (Exception e) { e.printStackTrace(); } // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { logger.info("Max memory capability resources in cluster: " + maxMem); } if (amMem > maxMem) { amMem = maxMem; logger.info("AM memory specified above threshold of cluster " + "Using maximum memory for AM container: " + amMem); } int maxVcores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { logger.info("Max vCores capability resources in cluster: " + maxVcores); } if (amCores > maxVcores) { amCores = maxVcores; logger.info("AM virtual cores specified above threshold of cluster " + "Using maximum virtual cores for AM container: " + amCores); } // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add("$JAVA_HOME/bin/java"); commands.add("-Xmx" + amMem + "m"); commands.add("runtime.starter.MPJAppMaster"); commands.add("--np"); commands.add(String.valueOf(np)); commands.add("--serverName"); commands.add(serverName); //server name commands.add("--ioServerPort"); commands.add(Integer.toString(serverPort)); //server port commands.add("--deviceName"); commands.add(deviceName); //device name commands.add("--className"); commands.add(className); //class name commands.add("--wdir"); commands.add(workingDirectory); //wdir commands.add("--psl"); commands.add(Integer.toString(psl)); //protocol switch limit commands.add("--wireUpPort"); commands.add(String.valueOf(TEMP_PORT)); //for sharing ports & rank commands.add("--wrapperPath"); commands.add(wrapperDest.toString());//MPJYarnWrapper.jar HDFS path commands.add("--userJarPath"); commands.add(userJarDest.toString());//User Jar File HDFS path commands.add("--mpjContainerPriority"); commands.add(mpjContainerPriority);// priority for mpj containers commands.add("--containerMem"); commands.add(containerMem); commands.add("--containerCores"); commands.add(containerCores); if (debugYarn) { commands.add("--debugYarn"); } if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } amContainer.setCommands(commands); //set commands // Setup local Resource for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(dest)); appMasterJar.setSize(destStatus.getLen()); appMasterJar.setTimestamp(destStatus.getModificationTime()); appMasterJar.setType(LocalResourceType.ARCHIVE); appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION); amContainer.setLocalResources(Collections.singletonMap("mpj-app-master.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster // Setting up the environment Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMem); capability.setVirtualCores(amCores); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(yarnQueue); // queue Priority priority = Priority.newInstance(amPriority); appContext.setPriority(priority); ApplicationId appId = appContext.getApplicationId(); //Adding ShutDown Hook Runtime.getRuntime().addShutdownHook(new KillYarnApp(appId, yarnClient)); // Submit application System.out.println("Submitting Application: " + appContext.getApplicationName() + "\n"); try { isRunning = true; yarnClient.submitApplication(appContext); } catch (Exception exp) { System.err.println("Error Submitting Application"); exp.printStackTrace(); } // np = number of processes , + 1 for Application Master container IOMessagesThread[] ioThreads = new IOMessagesThread[np + 1]; peers = new String[np]; socketList = new Vector<Socket>(); int wport = 0; int rport = 0; int rank = 0; // np + 1 IOThreads for (int i = 0; i < (np + 1); i++) { try { sock = servSock.accept(); //start IO thread to read STDOUT and STDERR from wrappers IOMessagesThread io = new IOMessagesThread(sock); ioThreads[i] = io; ioThreads[i].start(); } catch (Exception e) { System.err.println("Error accepting connection from peer socket.."); e.printStackTrace(); } } // Loop to read port numbers from Wrapper.java processes // and to create WRAPPER_INFO (containing all IPs and ports) String WRAPPER_INFO = "#Peer Information"; for (int i = np; i > 0; i--) { try { sock = infoSock.accept(); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); DataInputStream in = new DataInputStream(sock.getInputStream()); if (in.readUTF().startsWith("Sending Info")) { wport = in.readInt(); rport = in.readInt(); rank = in.readInt(); peers[rank] = ";" + sock.getInetAddress().getHostAddress() + "@" + rport + "@" + wport + "@" + rank; socketList.add(sock); } } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error accepting" + " connection from peer socket!"); e.printStackTrace(); } } for (int i = 0; i < np; i++) { WRAPPER_INFO += peers[i]; } // Loop to broadcast WRAPPER_INFO to all Wrappers for (int i = np; i > 0; i--) { try { sock = socketList.get(np - i); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); out.writeUTF(WRAPPER_INFO); out.flush(); sock.close(); } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error closing" + " connection from peer socket.."); e.printStackTrace(); } } try { infoSock.close(); } catch (IOException exp) { exp.printStackTrace(); } // wait for all IO Threads to complete for (int i = 0; i < (np + 1); i++) { ioThreads[i].join(); } isRunning = true; System.out.println("\nApplication Statistics!"); while (true) { appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); fStatus = appReport.getFinalApplicationStatus(); if (appState == YarnApplicationState.FINISHED) { isRunning = false; if (fStatus == FinalApplicationStatus.SUCCEEDED) { System.out.println("State: " + fStatus); } else { System.out.println("State: " + fStatus); } break; } else if (appState == YarnApplicationState.KILLED) { isRunning = false; System.out.println("State: " + appState); break; } else if (appState == YarnApplicationState.FAILED) { isRunning = false; System.out.println("State: " + appState); break; } Thread.sleep(100); } try { if (debugYarn) { logger.info("Cleaning the files from hdfs: "); logger.info("1) " + dest.toString()); logger.info("2) " + wrapperDest.toString()); logger.info("3) " + userJarDest.toString()); } fs.delete(dest); fs.delete(wrapperDest); fs.delete(userJarDest); } catch (IOException exp) { exp.printStackTrace(); } System.out.println("Application ID: " + appId + "\n" + "Application User: " + appReport.getUser() + "\n" + "RM Queue: " + appReport.getQueue() + "\n" + "Start Time: " + appReport.getStartTime() + "\n" + "Finish Time: " + appReport.getFinishTime()); }
From source file:tachyon.yarn.ApplicationMaster.java
License:Apache License
public void requestContainers() throws Exception { // Priority for Tachyon master and worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0);//from ww w. j a v a 2s.com // Resource requirements for master containers Resource masterResource = Records.newRecord(Resource.class); masterResource.setMemory(mMasterMemInMB); masterResource.setVirtualCores(mMasterCpu); String[] nodes = { mMasterAddress }; // Make container request for Tachyon master to ResourceManager boolean relaxLocality = true; if (!mMasterAddress.equals("localhost")) { relaxLocality = false; } ContainerRequest masterContainerAsk = new ContainerRequest(masterResource, nodes, null /* any racks */, priority, relaxLocality); LOG.info("Making resource request for Tachyon master: cpu {} memory {} MB on node {}", masterResource.getVirtualCores(), masterResource.getMemory(), mMasterAddress); mRMClient.addContainerRequest(masterContainerAsk); // Wait until Tachyon master container has been allocated while (!mMasterContainerAllocated) { Thread.sleep(1000); } // Resource requirements for master containers Resource workerResource = Records.newRecord(Resource.class); workerResource.setMemory(mWorkerMemInMB + mRamdiskMemInMB); workerResource.setVirtualCores(mWorkerCpu); // Make container requests for workers to ResourceManager for (int i = 0; i < mNumWorkers; i++) { ContainerRequest containerAsk = new ContainerRequest(workerResource, null /* any hosts */, null /* any racks */, priority); LOG.info("Making resource request for Tachyon worker {}: cpu {} memory {} MB on any nodes", i, workerResource.getVirtualCores(), workerResource.getMemory()); mRMClient.addContainerRequest(containerAsk); } // Wait until all Tachyon worker containers have been allocated while (mNumAllocatedWorkerContainers < mNumWorkers) { Thread.sleep(1000); } LOG.info("Master and workers are launched"); // Wait for 5 more seconds to avoid application unregistered before some container fully // launched. while (!mApplicationDone) { Thread.sleep(5000); } }
From source file:yarnkit.container.AbstractContainerLaunchParameters.java
License:Apache License
@Override public Resource getContainerResource(@Nonnull Resource capability) { Resource resource = Records.newRecord(Resource.class); resource.setVirtualCores(Math.min(capability.getVirtualCores(), getVirtualCores())); resource.setMemory(Math.min(capability.getMemory(), getMemory())); return resource; }
From source file:yrun.YarnRunner.java
License:Apache License
public void execute() throws IOException, YarnException, InterruptedException { LOG.info("Using application path [" + _installPath + "]"); Path jarPath = installThisJar(_installPath, _appJarFile); LOG.info("Driver installed [" + jarPath + "]"); List<Path> installedArchivePathList = install(_installPath, _archivePathList); for (Path p : installedArchivePathList) { LOG.info("Archive installed [" + p + "]"); }/* w ww . ja va 2 s . c o m*/ YarnRunnerArgs yarnRunnerArgs = new YarnRunnerArgs(); yarnRunnerArgs.setCommand(_command); Path argsPath = installThisArgs(_installPath, yarnRunnerArgs); final YarnClient client = YarnClient.createYarnClient(); _configuration.setInt("yarn.nodemanager.delete.debug-delay-sec", (int) TimeUnit.HOURS.toSeconds(1)); client.init(_configuration); client.start(); YarnClientApplication app = client.createApplication(); ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv, _appJarFile); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); { LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); localResources.put(jarPath.getName(), appMasterJar); } { LocalResource appMasterArgs = Records.newRecord(LocalResource.class); setupAppMasterArgs(argsPath, appMasterArgs); localResources.put(MASTER_JSON, appMasterArgs); } List<String> vargs = new ArrayList<String>(); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-Xmx256m"); vargs.add("-Djava.net.preferIPv4Stack=true"); vargs.add(YarnRunnerApplicationMaster.class.getName()); String strCommand = "(echo ENV && set && echo CURRENT_DIR_LISTING && ls -la && echo PWD && pwd && (" + StringUtils.join(" ", vargs) + "))"; strCommand += " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"; strCommand += " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; LOG.debug("Application Master command [" + strCommand + "]"); amContainer.setCommands(Collections.singletonList(strCommand)); amContainer.setLocalResources(localResources); amContainer.setEnvironment(appMasterEnv); Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(_yarnName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); if (_queue != null) { appContext.setQueue(_queue); } appContext.setApplicationType("yrun"); ApplicationId appId = appContext.getApplicationId(); AtomicBoolean shutdown = new AtomicBoolean(); if (!_isDaemon) { addShutdownHook(client, appId, shutdown); } LOG.info("Submitting application with id [" + appId + "]"); client.submitApplication(appContext); ApplicationReport report; YarnApplicationState state; do { report = client.getApplicationReport(appId); state = report.getYarnApplicationState(); if (state == YarnApplicationState.RUNNING) { if (_isDaemon) { LOG.info("Application is running. This is a daemon application driver program exiting."); return; } } Thread.sleep(100); } while (isNoLongerRunning(state)); shutdown.set(true); LOG.info("Application has finished with state [" + state + "]"); }
From source file:yrun.YarnRunnerApplicationMaster.java
License:Apache License
public void run(String[] args) throws Exception { JsonParser parser = new JsonParser(); JsonElement element = parser.parse(new FileReader(YarnRunner.MASTER_JSON)); LOG.info("Json [" + element + "]"); JsonObject jsonObject = (JsonObject) element; int priority = jsonObject.get("priority").getAsInt(); int numberOfContainers = jsonObject.get("numberOfContainers").getAsInt(); int memory = jsonObject.get("memory").getAsInt(); int vCores = jsonObject.get("vCores").getAsInt(); String command = jsonObject.get("command").getAsString(); // startHttpServer(); // InetSocketAddress address = _server.getAddress(); // LOG.info("Http server started at [" + address + "]"); // String appHostName = "app-host-name"; // int appHostPort = address.getPort(); // String appTrackingUrl = "http://" + address.getHostName() + ":" + // appHostPort + "/"; // LOG.info("App Tracking Url [" + appTrackingUrl + "]"); // Initialize clients to ResourceManager and NodeManagers Configuration conf = new YarnConfiguration(); AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf);//ww w . j a va 2 s .co m rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager LOG.info("Register Application Master 0"); String appHostName = ""; int appHostPort = 0; String appTrackingUrl = ""; rmClient.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl); LOG.info("Register Application Master 1"); // Priority for worker containers - priorities are intra-application Priority priorityRecord = Records.newRecord(Priority.class); priorityRecord.setPriority(priority); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); capability.setVirtualCores(vCores); // Make container requests to ResourceManager for (int i = 0; i < numberOfContainers; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priorityRecord); LOG.info("Making resource request for [" + i + "]"); rmClient.addContainerRequest(containerAsk); } // Obtain allocated containers, launch and check for responses int responseId = 0; int completedContainers = 0; long lastReport = 0; List<Container> containers = new ArrayList<Container>(); while (completedContainers < numberOfContainers) { if (lastReport + TimeUnit.SECONDS.toNanos(5) < System.nanoTime()) { for (Container container : containers) { ContainerId containerId = container.getId(); NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = nmClient.getContainerStatus(containerId, nodeId); LOG.info("NodeId [" + nodeId + "] Container Status [" + containerStatus + "]"); // Figure out } lastReport = System.nanoTime(); } AllocateResponse response = rmClient.allocate(responseId++); for (Container container : response.getAllocatedContainers()) { containers.add(container); // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setCommands( Collections.singletonList(command + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout2" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); LOG.info("Launching container " + container.getId()); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { completedContainers++; LOG.info("Completed container " + status.getContainerId()); } Thread.sleep(100); } // _server.stop(0); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }