List of usage examples for org.apache.hadoop.yarn.api.records NodeReport getCapability
@Public @Stable public abstract Resource getCapability();
Resource on the node. From source file:org.apache.flink.yarn.FlinkYarnClientBase.java
License:Apache License
@Override public String getClusterDescription() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); ps.append("NodeManagers in the Cluster " + metrics.getNumNodeManagers()); List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); final String format = "|%-16s |%-16s %n"; ps.printf("|Property |Value %n"); ps.println("+---------------------------------------+"); int totalMemory = 0; int totalCores = 0; for (NodeReport rep : nodes) { final Resource res = rep.getCapability(); totalMemory += res.getMemory();//www . j a v a 2 s .co m totalCores += res.getVirtualCores(); ps.format(format, "NodeID", rep.getNodeId()); ps.format(format, "Memory", res.getMemory() + " MB"); ps.format(format, "vCores", res.getVirtualCores()); ps.format(format, "HealthReport", rep.getHealthReport()); ps.format(format, "Containers", rep.getNumContainers()); ps.println("+---------------------------------------+"); } ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores); List<QueueInfo> qInfo = yarnClient.getAllQueues(); for (QueueInfo q : qInfo) { ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " + q.getMaximumCapacity() + " Applications: " + q.getApplications().size()); } yarnClient.stop(); return baos.toString(); }
From source file:org.apache.giraph.yarn.GiraphYarnClient.java
License:Apache License
/** * Utility to make sure we have the cluster resources we need to run this * job. If they are not available, we should die here before too much setup. * @param cluster the GetNewApplicationResponse from the YARN RM. *///from w w w .j a v a 2s . c om private void checkPerNodeResourcesAvailable(final GetNewApplicationResponse cluster) throws YarnException, IOException { // are there enough containers to go around for our Giraph job? List<NodeReport> nodes = null; long totalAvailable = 0; try { nodes = yarnClient.getNodeReports(NodeState.RUNNING); } catch (YarnException yre) { throw new RuntimeException("GiraphYarnClient could not connect with " + "the YARN ResourceManager to determine the number of available " + "application containers.", yre); } for (NodeReport node : nodes) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress " + node.getHttpAddress() + ", nodeRackName " + node.getRackName() + ", nodeNumContainers " + node.getNumContainers()); totalAvailable += node.getCapability().getMemory(); } // 1 master + all workers in -w command line arg final int workers = giraphConf.getMaxWorkers() + 1; checkAndAdjustPerTaskHeapSize(cluster); final long totalAsk = giraphConf.getYarnTaskHeapMb() * workers; if (totalAsk > totalAvailable) { throw new IllegalStateException("Giraph's estimated cluster heap " + totalAsk + "MB ask is greater than the current available cluster " + "heap of " + totalAvailable + "MB. Aborting Job."); } }
From source file:org.apache.reef.runtime.yarn.driver.YarnContainerManager.java
License:Apache License
private void onNodeReport(final NodeReport nodeReport) { LOG.log(Level.FINE, "Send node descriptor: {0}", nodeReport); this.reefEventHandlers.onNodeDescriptor(NodeDescriptorProto.newBuilder() .setIdentifier(nodeReport.getNodeId().toString()).setHostName(nodeReport.getNodeId().getHost()) .setPort(nodeReport.getNodeId().getPort()).setMemorySize(nodeReport.getCapability().getMemory()) .setRackName(nodeReport.getRackName()).build()); }
From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java
License:Apache License
public static double getClusterUtilization() throws IOException { double util = 0; try {/*w w w. jav a 2s. co m*/ if (_client == null) _client = createYarnClient(); List<NodeReport> nodesReport = _client.getNodeReports(); double maxMem = 0; double currMem = 0; long maxCores = 0; long currCores = 0; for (NodeReport node : nodesReport) { Resource max = node.getCapability(); Resource used = node.getUsed(); maxMem += max.getMemory(); currMem += used.getMemory(); maxCores += max.getVirtualCores(); currCores += used.getVirtualCores(); } util = Math.max(Math.min(1, currMem / maxMem), //memory util Math.min(1, (double) currCores / maxCores)); //vcore util } catch (Exception ex) { throw new IOException(ex); } return util; }
From source file:org.apache.sysml.yarn.ropt.YarnClusterAnalyzer.java
License:Apache License
/** * Analyzes properties of Yarn cluster and Hadoop configurations. * //from w w w . j a v a2s .c o m * @param yarnClient hadoop yarn client * @param conf hadoop yarn configuration * @param verbose output info to standard output */ public static void analyzeYarnCluster(YarnClient yarnClient, YarnConfiguration conf, boolean verbose) { try { List<NodeReport> nodesReport = yarnClient.getNodeReports(); if (verbose) System.out.println("There are " + nodesReport.size() + " nodes in the cluster"); if (nodesReport.isEmpty()) throw new YarnException("There are zero available nodes in the yarn cluster"); nodesMaxPhySorted = new ArrayList<>(nodesReport.size()); clusterTotalMem = 0; clusterTotalCores = 0; clusterTotalNodes = 0; minimumMRContainerPhyMB = -1; for (NodeReport node : nodesReport) { Resource resource = node.getCapability(); Resource used = node.getUsed(); if (used == null) used = Resource.newInstance(0, 0); int mb = resource.getMemory(); int cores = resource.getVirtualCores(); if (mb <= 0) throw new YarnException("A node has non-positive memory " + mb); int myMinMRPhyMB = mb / cores / CPU_HYPER_FACTOR; if (minimumMRContainerPhyMB < myMinMRPhyMB) minimumMRContainerPhyMB = myMinMRPhyMB; // minimumMRContainerPhyMB needs to be the largest among the mins clusterTotalMem += (long) mb * 1024 * 1024; nodesMaxPhySorted.add((long) mb * 1024 * 1024); clusterTotalCores += cores; clusterTotalNodes++; if (verbose) System.out.println("\t" + node.getNodeId() + " has " + mb + " MB (" + used.getMemory() + " MB used) memory and " + resource.getVirtualCores() + " (" + used.getVirtualCores() + " used) cores"); } Collections.sort(nodesMaxPhySorted, Collections.reverseOrder()); nodesMaxBudgetSorted = new ArrayList<>(nodesMaxPhySorted.size()); for (int i = 0; i < nodesMaxPhySorted.size(); i++) nodesMaxBudgetSorted.add(ResourceOptimizer.phyToBudget(nodesMaxPhySorted.get(i))); _remotePar = nodesReport.size(); if (_remotePar == 0) throw new YarnException("There are no available nodes in the yarn cluster"); // Now get the default cluster settings _remoteMRSortMem = (1024 * 1024) * conf.getLong(MRConfigurationNames.MR_TASK_IO_SORT_MB, 100); //100MB //handle jvm max mem (map mem budget is relevant for map-side distcache and parfor) //(for robustness we probe both: child and map configuration parameters) String javaOpts1 = conf.get(MRConfigurationNames.MR_CHILD_JAVA_OPTS); //internally mapred/mapreduce synonym String javaOpts2 = conf.get(MRConfigurationNames.MR_MAP_JAVA_OPTS, null); //internally mapred/mapreduce synonym String javaOpts3 = conf.get(MRConfigurationNames.MR_REDUCE_JAVA_OPTS, null); //internally mapred/mapreduce synonym if (javaOpts2 != null) //specific value overrides generic _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts2); else _remoteJVMMaxMemMap = extractMaxMemoryOpt(javaOpts1); if (javaOpts3 != null) //specific value overrides generic _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts3); else _remoteJVMMaxMemReduce = extractMaxMemoryOpt(javaOpts1); //HDFS blocksize String blocksize = conf.get(MRConfigurationNames.DFS_BLOCKSIZE, "134217728"); _blocksize = Long.parseLong(blocksize); minimalPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); maximumPhyAllocate = (long) 1024 * 1024 * conf.getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); mrAMPhy = (long) conf.getInt(MRConfigurationNames.YARN_APP_MR_AM_RESOURCE_MB, 1536) * 1024 * 1024; } catch (Exception e) { throw new RuntimeException("Unable to analyze yarn cluster ", e); } /* * This is for AppMaster to query available resource in the cluster during heartbeat * AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); AllocateResponse response = rmClient.allocate(0); int nodeCount = response.getNumClusterNodes(); Resource resource = response.getAvailableResources(); List<NodeReport> nodeUpdate = response.getUpdatedNodes(); LOG.info("This is a " + nodeCount + " node cluster with totally " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); LOG.info(nodereport.size() + " updatedNode reports received"); for (NodeReport node : nodeUpdate) { resource = node.getCapability(); LOG.info(node.getNodeId() + " updated with " + resource.getMemory() + " memory and " + resource.getVirtualCores() + " cores"); }*/ }
From source file:org.apache.twill.yarn.PlacementPolicyTestRun.java
License:Apache License
/** * Verify the cluster configuration (number and capability of node managers) required for the tests. *//* www .ja va 2 s .c om*/ @BeforeClass public static void verifyClusterCapability() throws InterruptedException { // Ignore verifications if it is running against older Hadoop versions which does not support blacklists. Assume.assumeTrue(YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_22)); // All runnables in this test class use same resource specification for the sake of convenience. resource = ResourceSpecification.Builder.with().setVirtualCores(RUNNABLE_CORES) .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA).build(); twoInstancesResource = ResourceSpecification.Builder.with().setVirtualCores(RUNNABLE_CORES) .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA).setInstances(2).build(); // The tests need exactly three NodeManagers in the cluster. int trials = 0; while (trials++ < 20) { try { nodeReports = TWILL_TESTER.getNodeReports(); if (nodeReports != null && nodeReports.size() == 3) { break; } } catch (Exception e) { LOG.error("Failed to get node reports", e); } LOG.warn("NodeManagers != 3. {}", nodeReports); TimeUnit.SECONDS.sleep(1); } // All NodeManagers should have enough capacity available to accommodate at least two runnables. for (NodeReport nodeReport : nodeReports) { Resource capability = nodeReport.getCapability(); Resource used = nodeReport.getUsed(); Assert.assertNotNull(capability); if (used != null) { Assert.assertTrue(2 * resource.getMemorySize() < capability.getMemory() - used.getMemory()); } else { Assert.assertTrue(2 * resource.getMemorySize() < capability.getMemory()); } } }
From source file:org.huahinframework.manager.rest.service.ApplicationService.java
License:Apache License
@Path("/cluster") @GET//from ww w . j a v a 2s. c om @Produces(MediaType.APPLICATION_JSON) public JSONObject getCluster() { JSONObject jsonObject = new JSONObject(); try { GetClusterMetricsRequest metricsRequest = recordFactory .newRecordInstance(GetClusterMetricsRequest.class); GetClusterMetricsResponse metricsResponse = applicationsManager.getClusterMetrics(metricsRequest); jsonObject.put(Response.NUM_NODE_MANAGERS, metricsResponse.getClusterMetrics().getNumNodeManagers()); GetClusterNodesRequest nodeRequest = recordFactory.newRecordInstance(GetClusterNodesRequest.class); GetClusterNodesResponse nodeResponse = applicationsManager.getClusterNodes(nodeRequest); List<JSONObject> reports = new ArrayList<JSONObject>(); for (NodeReport report : nodeResponse.getNodeReports()) { JSONObject nr = new JSONObject(); nr.put(Response.HTTP_ADDRESS, report.getHttpAddress()); nr.put(Response.NUM_CONTAINERS, report.getNumContainers()); nr.put(Response.RACK_NAME, report.getRackName()); nr.put(Response.CAPABILITY, report.getCapability().getMemory()); nr.put(Response.HEALTH_REPORT, report.getNodeHealthStatus().getHealthReport()); nr.put(Response.IS_NODE_HEALTHY, report.getNodeHealthStatus().getIsNodeHealthy()); nr.put(Response.LAST_HEALTH_REPORT_TIME, new Date(report.getNodeHealthStatus().getLastHealthReportTime())); nr.put(Response.NODE_ID, report.getNodeId()); nr.put(Response.NODE_STATE, report.getNodeState()); nr.put(Response.NODE_STATE, report.getNodeState()); nr.put(Response.USED, report.getUsed()); reports.add(nr); } jsonObject.put(Response.NODES, reports); } catch (Exception e) { e.printStackTrace(); log.error(e); Map<String, String> status = new HashMap<String, String>(); status.put(Response.STATUS, e.getMessage()); jsonObject = new JSONObject(status); } return jsonObject; }
From source file:runtime.starter.MPJYarnClient.java
License:Open Source License
public void run() throws Exception { Map<String, String> map = System.getenv(); try {/* w w w. j a v a 2s. c o m*/ mpjHomeDir = map.get("MPJ_HOME"); if (mpjHomeDir == null) { throw new Exception("[MPJRun.java]:MPJ_HOME environment found.."); } } catch (Exception exc) { System.out.println("[MPJRun.java]:" + exc.getMessage()); exc.printStackTrace(); return; } // Copy the application master jar to HDFS // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); /* Path dataset = new Path(fs.getHomeDirectory(),"/dataset"); FileStatus datasetFile = fs.getFileStatus(dataset); BlockLocation myBlocks [] = fs.getFileBlockLocations(datasetFile,0,datasetFile.getLen()); for(BlockLocation b : myBlocks){ System.out.println("\n--------------------"); System.out.println("Length "+b.getLength()); for(String host : b.getHosts()){ System.out.println("host "+host); } } */ Path source = new Path(mpjHomeDir + "/lib/mpj-app-master.jar"); String pathSuffix = hdfsFolder + "mpj-app-master.jar"; Path dest = new Path(fs.getHomeDirectory(), pathSuffix); if (debugYarn) { logger.info("Uploading mpj-app-master.jar to: " + dest.toString()); } fs.copyFromLocalFile(false, true, source, dest); FileStatus destStatus = fs.getFileStatus(dest); Path wrapperSource = new Path(mpjHomeDir + "/lib/mpj-yarn-wrapper.jar"); String wrapperSuffix = hdfsFolder + "mpj-yarn-wrapper.jar"; Path wrapperDest = new Path(fs.getHomeDirectory(), wrapperSuffix); if (debugYarn) { logger.info("Uploading mpj-yarn-wrapper.jar to: " + wrapperDest.toString()); } fs.copyFromLocalFile(false, true, wrapperSource, wrapperDest); Path userJar = new Path(jarPath); String userJarSuffix = hdfsFolder + "user-code.jar"; Path userJarDest = new Path(fs.getHomeDirectory(), userJarSuffix); if (debugYarn) { logger.info("Uploading user-code.jar to: " + userJarDest.toString()); } fs.copyFromLocalFile(false, true, userJar, userJarDest); YarnConfiguration conf = new YarnConfiguration(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); if (debugYarn) { YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); logger.info("\nNodes Information"); logger.info("Number of NM: " + metrics.getNumNodeManagers() + "\n"); List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport n : nodeReports) { logger.info("NodeId: " + n.getNodeId()); logger.info("RackName: " + n.getRackName()); logger.info("Total Memory: " + n.getCapability().getMemory()); logger.info("Used Memory: " + n.getUsed().getMemory()); logger.info("Total vCores: " + n.getCapability().getVirtualCores()); logger.info("Used vCores: " + n.getUsed().getVirtualCores() + "\n"); } } logger.info("Creating server socket at HOST " + serverName + " PORT " + serverPort + " \nWaiting for " + np + " processes to connect..."); // Creating a server socket for incoming connections try { servSock = new ServerSocket(serverPort); infoSock = new ServerSocket(); TEMP_PORT = findPort(infoSock); } catch (Exception e) { e.printStackTrace(); } // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { logger.info("Max memory capability resources in cluster: " + maxMem); } if (amMem > maxMem) { amMem = maxMem; logger.info("AM memory specified above threshold of cluster " + "Using maximum memory for AM container: " + amMem); } int maxVcores = appResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { logger.info("Max vCores capability resources in cluster: " + maxVcores); } if (amCores > maxVcores) { amCores = maxVcores; logger.info("AM virtual cores specified above threshold of cluster " + "Using maximum virtual cores for AM container: " + amCores); } // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add("$JAVA_HOME/bin/java"); commands.add("-Xmx" + amMem + "m"); commands.add("runtime.starter.MPJAppMaster"); commands.add("--np"); commands.add(String.valueOf(np)); commands.add("--serverName"); commands.add(serverName); //server name commands.add("--ioServerPort"); commands.add(Integer.toString(serverPort)); //server port commands.add("--deviceName"); commands.add(deviceName); //device name commands.add("--className"); commands.add(className); //class name commands.add("--wdir"); commands.add(workingDirectory); //wdir commands.add("--psl"); commands.add(Integer.toString(psl)); //protocol switch limit commands.add("--wireUpPort"); commands.add(String.valueOf(TEMP_PORT)); //for sharing ports & rank commands.add("--wrapperPath"); commands.add(wrapperDest.toString());//MPJYarnWrapper.jar HDFS path commands.add("--userJarPath"); commands.add(userJarDest.toString());//User Jar File HDFS path commands.add("--mpjContainerPriority"); commands.add(mpjContainerPriority);// priority for mpj containers commands.add("--containerMem"); commands.add(containerMem); commands.add("--containerCores"); commands.add(containerCores); if (debugYarn) { commands.add("--debugYarn"); } if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } amContainer.setCommands(commands); //set commands // Setup local Resource for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(dest)); appMasterJar.setSize(destStatus.getLen()); appMasterJar.setTimestamp(destStatus.getModificationTime()); appMasterJar.setType(LocalResourceType.ARCHIVE); appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION); amContainer.setLocalResources(Collections.singletonMap("mpj-app-master.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster // Setting up the environment Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMem); capability.setVirtualCores(amCores); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue(yarnQueue); // queue Priority priority = Priority.newInstance(amPriority); appContext.setPriority(priority); ApplicationId appId = appContext.getApplicationId(); //Adding ShutDown Hook Runtime.getRuntime().addShutdownHook(new KillYarnApp(appId, yarnClient)); // Submit application System.out.println("Submitting Application: " + appContext.getApplicationName() + "\n"); try { isRunning = true; yarnClient.submitApplication(appContext); } catch (Exception exp) { System.err.println("Error Submitting Application"); exp.printStackTrace(); } // np = number of processes , + 1 for Application Master container IOMessagesThread[] ioThreads = new IOMessagesThread[np + 1]; peers = new String[np]; socketList = new Vector<Socket>(); int wport = 0; int rport = 0; int rank = 0; // np + 1 IOThreads for (int i = 0; i < (np + 1); i++) { try { sock = servSock.accept(); //start IO thread to read STDOUT and STDERR from wrappers IOMessagesThread io = new IOMessagesThread(sock); ioThreads[i] = io; ioThreads[i].start(); } catch (Exception e) { System.err.println("Error accepting connection from peer socket.."); e.printStackTrace(); } } // Loop to read port numbers from Wrapper.java processes // and to create WRAPPER_INFO (containing all IPs and ports) String WRAPPER_INFO = "#Peer Information"; for (int i = np; i > 0; i--) { try { sock = infoSock.accept(); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); DataInputStream in = new DataInputStream(sock.getInputStream()); if (in.readUTF().startsWith("Sending Info")) { wport = in.readInt(); rport = in.readInt(); rank = in.readInt(); peers[rank] = ";" + sock.getInetAddress().getHostAddress() + "@" + rport + "@" + wport + "@" + rank; socketList.add(sock); } } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error accepting" + " connection from peer socket!"); e.printStackTrace(); } } for (int i = 0; i < np; i++) { WRAPPER_INFO += peers[i]; } // Loop to broadcast WRAPPER_INFO to all Wrappers for (int i = np; i > 0; i--) { try { sock = socketList.get(np - i); DataOutputStream out = new DataOutputStream(sock.getOutputStream()); out.writeUTF(WRAPPER_INFO); out.flush(); sock.close(); } catch (Exception e) { System.err.println("[MPJYarnClient.java]: Error closing" + " connection from peer socket.."); e.printStackTrace(); } } try { infoSock.close(); } catch (IOException exp) { exp.printStackTrace(); } // wait for all IO Threads to complete for (int i = 0; i < (np + 1); i++) { ioThreads[i].join(); } isRunning = true; System.out.println("\nApplication Statistics!"); while (true) { appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); fStatus = appReport.getFinalApplicationStatus(); if (appState == YarnApplicationState.FINISHED) { isRunning = false; if (fStatus == FinalApplicationStatus.SUCCEEDED) { System.out.println("State: " + fStatus); } else { System.out.println("State: " + fStatus); } break; } else if (appState == YarnApplicationState.KILLED) { isRunning = false; System.out.println("State: " + appState); break; } else if (appState == YarnApplicationState.FAILED) { isRunning = false; System.out.println("State: " + appState); break; } Thread.sleep(100); } try { if (debugYarn) { logger.info("Cleaning the files from hdfs: "); logger.info("1) " + dest.toString()); logger.info("2) " + wrapperDest.toString()); logger.info("3) " + userJarDest.toString()); } fs.delete(dest); fs.delete(wrapperDest); fs.delete(userJarDest); } catch (IOException exp) { exp.printStackTrace(); } System.out.println("Application ID: " + appId + "\n" + "Application User: " + appReport.getUser() + "\n" + "RM Queue: " + appReport.getQueue() + "\n" + "Start Time: " + appReport.getStartTime() + "\n" + "Finish Time: " + appReport.getFinishTime()); }
From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java
License:Open Source License
@Override public PBSNodeStatus[] getNodesStatus() throws Exception { //first use the container reports of all running jobs to get a picture of the hosts in use //for each job TIntObjectHashMap<List<ContainerId>> job2con = getAllActiveContainers(); final Map<String, TIntArrayList> node2job = new HashMap<String, TIntArrayList>(); job2con.forEachEntry(new TIntObjectProcedure<List<ContainerId>>() { @Override/*from www . j a v a 2 s . c om*/ public boolean execute(int jobId, List<ContainerId> containerList) { for (ContainerId cid : containerList) { try { ContainerReport cr = yClient.getContainerReport(cid); String hostname = cr.getAssignedNode().getHost(); TIntArrayList jobs = node2job.get(hostname); if (jobs == null) node2job.put(hostname, jobs = new TIntArrayList()); jobs.add(jobId); } catch (Exception e) { throw new RuntimeException(e); } } return true; } }); List<NodeReport> nodeReports = yClient.getNodeReports(); PBSNodeStatus[] rtr = new PBSNodeStatus[nodeReports.size()]; for (int i = 0; i < rtr.length; i++) { final NodeReport node = nodeReports.get(i); String hostname = node.getNodeId().getHost(); String yarnState = node.getNodeState().toString(); String rack = node.getRackName(); String tracker = node.getHttpAddress(); int numContainers = node.getNumContainers(); int numProcs = node.getCapability().getVirtualCores(); TIntArrayList jobList = node2job.get(hostname); int[] jobs; if (jobList == null) jobs = new int[0]; else jobs = jobList.toArray(); String state = "free"; if (numContainers >= numProcs) state = "busy"; StringBuilder status = new StringBuilder(); status.append("capacity=" + node.getCapability().toString()); status.append(",used=" + node.getUsed().toString()); rtr[i] = new PBSNodeStatus(hostname, state, status.toString(), jobs, tracker, node.getHealthReport(), rack, yarnState, numProcs, node.getNodeLabels()); } return rtr; }