List of usage examples for org.apache.hadoop.yarn.api.records ContainerReport getAssignedNode
@Public @Unstable public abstract NodeId getAssignedNode();
NodeId where container is running. From source file:io.hops.hopsworks.common.admin.llap.LlapClusterFacade.java
License:Open Source License
public List<String> getLlapHosts() { ArrayList<String> hosts = new ArrayList<>(); if (!isClusterUp() || isClusterStarting()) { return hosts; }// w ww .j a va 2 s . c om // The cluster is app, so the appId exists String llapAppID = variablesFacade.getVariableValue(Settings.VARIABLE_LLAP_APP_ID); ApplicationId appId = ApplicationId.fromString(llapAppID); YarnClient yarnClient = yarnClientService.getYarnClientSuper(settings.getConfiguration()).getYarnClient(); try { List<ApplicationAttemptReport> attempts = yarnClient.getApplicationAttempts(appId); ApplicationAttemptReport current = null; for (ApplicationAttemptReport attempt : attempts) { // Only if the app is running the metrics are available if (attempt.getYarnApplicationAttemptState() == YarnApplicationAttemptState.RUNNING) { current = attempt; break; } } if (current == null) { return hosts; } List<ContainerReport> containerReports = yarnClient.getContainers(current.getApplicationAttemptId()); // For all the new/running containers, which are not the application master, get the host for (ContainerReport containerReport : containerReports) { // Only if the container is running the metrics are available if (containerReport.getContainerState() == ContainerState.RUNNING && !containerReport.getContainerId().equals(current.getAMContainerId())) { hosts.add(containerReport.getAssignedNode().getHost()); } } } catch (IOException | YarnException ex) { logger.log(Level.SEVERE, "Couldn't retrieve the containers for LLAP cluster", ex); } finally { try { yarnClient.close(); } catch (IOException ex) { } } return hosts; }
From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java
License:Open Source License
@Override public PBSJobStatusLight getJobStatus(int jobId, int requestType) throws Exception { if (requestType > 5 || requestType < 0) throw new IllegalArgumentException("requestType must be [0,1,2,3,4,5]"); char state = '*'; if (!jobArray.containsKey(jobId)) state = '?'; final JobInformation ji = jobArray.get(jobId); final PBSJob job = ji != null ? ji.jobSpec : null; YarnClientService kittenClient = ji != null ? ji.kitten : null; ApplicationReport appReport = null;// www. ja v a 2 s . c om if (kittenClient == null || (appReport = kittenClient.getApplicationReport()) == null) { state = '?'; if (jobHolds.get(jobId) != null) { state = 'H'; } } else { YarnApplicationState appState = appReport.getYarnApplicationState(); if (kittenClient.isApplicationFinished()) state = 'E'; else switch (appState) { case NEW: case NEW_SAVING: case ACCEPTED: case SUBMITTED: state = 'Q'; break; case FAILED: case KILLED: case FINISHED: state = 'E'; break; case RUNNING: state = 'R'; break; default: state = '?'; break; } } String timeUse = appReport == null ? "0" : Utils.makeTime(appReport.getApplicationResourceUsageReport().getVcoreSeconds()); String appURL = appReport == null ? "" : appReport.getTrackingUrl(); PBSJobStatusLight rtr = null; String nodes = null; List<ContainerReport> cReports = null; String appId = null; if (requestType == 0) { rtr = new PBSJobStatusLight(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL); } else if (requestType == 4) { checkOwnerOrRoot(ji); JobInteractiveInfo jii = ji != null ? ji.interactive : null; rtr = new PBSJobStatusInteractive(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, jii != null ? jii.hostname : null, jii != null ? jii.port : -1, jii != null ? jii.secret : null); } else if (requestType == 5) { checkOwnerOrRoot(ji); JobDistributedInfo jid = ji != null ? ji.distributed : null; String secret = jid != null ? jid.secret : null; String[] hostnames = jid != null ? jid.hostnames.toArray(new String[0]) : null; int[] ports = jid != null ? jid.ports.toArray() : null; rtr = new PBSJobStatusDistributed(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, hostnames, ports, secret); } //we need the nodes also else if (requestType >= 1) { if (kittenClient != null) { ApplicationId aid = kittenClient.getApplicationReport().getApplicationId(); appId = aid.toString(); List<ApplicationAttemptReport> aaids = yClient.getApplicationAttempts(aid); ApplicationAttemptId aaid = aaids.get(aaids.size() - 1).getApplicationAttemptId(); cReports = yClient.getContainers(aaid); StringBuilder sNodes = new StringBuilder(); if (cReports.size() > 0) { for (ContainerReport cReport : cReports) { if (cReport.getContainerState() == ContainerState.RUNNING) { sNodes.append(cReport.getAssignedNode().getHost()); sNodes.append("+"); } } //remove trailing , sNodes.setLength(sNodes.length() - 1); } nodes = sNodes.toString(); } if (requestType == 1) { rtr = new PBSJobStatusNodes(jobId, job.getArrayTaskIds() != null, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, nodes); } else if (requestType == 2) { String[] tContainers; if (job != null) { tContainers = job.getArrayTaskIds() != null ? ji.array2Container.values(new String[0]) : new String[] { ji.taskContainerId }; } else { tContainers = new String[0]; } String trackingURL = appReport != null ? appReport.getTrackingUrl() : null; rtr = new PBSJobStatusFat(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, nodes, ji != null ? ji.jobSpec : null, ji != null ? ji.masterContainerId : null, tContainers, trackingURL, appId); } else if (requestType == 3) { int[] arrayIds = job != null ? JobUtils.getTaskArrayItems(job.getArrayTaskIds()) : new int[0]; if (arrayIds == null) arrayIds = new int[0]; char[] states = new char[arrayIds.length]; //String[] walltime = new String[arrayIds.length]; int i = 0; for (int arid : arrayIds) { String containerStatus = ji.array2Container.get(arid); if (containerStatus == null) states[i] = 'Q'; else if (containerStatus.equals("DONE")) states[i] = 'C'; else if (containerStatus.equals("ABORTED")) states[i] = 'C'; else { states[i] = 'R'; // ContainerId c = ContainerId.fromString(containerStatus); // for(ContainerReport cReport : cReports) // { // if (cReport.getContainerId().equals(c) // { // walltime[i] = cReport. // } } i++; } rtr = new PBSJobArrayStatusLight(jobId, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, arrayIds, states); } else { //this should not be reached. throw new IllegalArgumentException("Bad requestType"); } } return rtr; }
From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java
License:Open Source License
@Override public PBSNodeStatus[] getNodesStatus() throws Exception { //first use the container reports of all running jobs to get a picture of the hosts in use //for each job TIntObjectHashMap<List<ContainerId>> job2con = getAllActiveContainers(); final Map<String, TIntArrayList> node2job = new HashMap<String, TIntArrayList>(); job2con.forEachEntry(new TIntObjectProcedure<List<ContainerId>>() { @Override/*from w w w .j ava 2 s . c om*/ public boolean execute(int jobId, List<ContainerId> containerList) { for (ContainerId cid : containerList) { try { ContainerReport cr = yClient.getContainerReport(cid); String hostname = cr.getAssignedNode().getHost(); TIntArrayList jobs = node2job.get(hostname); if (jobs == null) node2job.put(hostname, jobs = new TIntArrayList()); jobs.add(jobId); } catch (Exception e) { throw new RuntimeException(e); } } return true; } }); List<NodeReport> nodeReports = yClient.getNodeReports(); PBSNodeStatus[] rtr = new PBSNodeStatus[nodeReports.size()]; for (int i = 0; i < rtr.length; i++) { final NodeReport node = nodeReports.get(i); String hostname = node.getNodeId().getHost(); String yarnState = node.getNodeState().toString(); String rack = node.getRackName(); String tracker = node.getHttpAddress(); int numContainers = node.getNumContainers(); int numProcs = node.getCapability().getVirtualCores(); TIntArrayList jobList = node2job.get(hostname); int[] jobs; if (jobList == null) jobs = new int[0]; else jobs = jobList.toArray(); String state = "free"; if (numContainers >= numProcs) state = "busy"; StringBuilder status = new StringBuilder(); status.append("capacity=" + node.getCapability().toString()); status.append(",used=" + node.getUsed().toString()); rtr[i] = new PBSNodeStatus(hostname, state, status.toString(), jobs, tracker, node.getHealthReport(), rack, yarnState, numProcs, node.getNodeLabels()); } return rtr; }