List of usage examples for org.apache.hadoop.yarn.api.records ContainerState RUNNING
ContainerState RUNNING
To view the source code for org.apache.hadoop.yarn.api.records ContainerState RUNNING.
Click Source Link
From source file:io.hops.hopsworks.common.admin.llap.LlapClusterFacade.java
License:Open Source License
public List<String> getLlapHosts() { ArrayList<String> hosts = new ArrayList<>(); if (!isClusterUp() || isClusterStarting()) { return hosts; }//from w w w . j av a 2 s . co m // The cluster is app, so the appId exists String llapAppID = variablesFacade.getVariableValue(Settings.VARIABLE_LLAP_APP_ID); ApplicationId appId = ApplicationId.fromString(llapAppID); YarnClient yarnClient = yarnClientService.getYarnClientSuper(settings.getConfiguration()).getYarnClient(); try { List<ApplicationAttemptReport> attempts = yarnClient.getApplicationAttempts(appId); ApplicationAttemptReport current = null; for (ApplicationAttemptReport attempt : attempts) { // Only if the app is running the metrics are available if (attempt.getYarnApplicationAttemptState() == YarnApplicationAttemptState.RUNNING) { current = attempt; break; } } if (current == null) { return hosts; } List<ContainerReport> containerReports = yarnClient.getContainers(current.getApplicationAttemptId()); // For all the new/running containers, which are not the application master, get the host for (ContainerReport containerReport : containerReports) { // Only if the container is running the metrics are available if (containerReport.getContainerState() == ContainerState.RUNNING && !containerReport.getContainerId().equals(current.getAMContainerId())) { hosts.add(containerReport.getAssignedNode().getHost()); } } } catch (IOException | YarnException ex) { logger.log(Level.SEVERE, "Couldn't retrieve the containers for LLAP cluster", ex); } finally { try { yarnClient.close(); } catch (IOException ex) { } } return hosts; }
From source file:io.hops.util.DBUtility.java
License:Apache License
public static RMNode processHopRMNodeCompsForScheduler(RMNodeComps hopRMNodeComps, RMContext rmContext) throws InvalidProtocolBufferException { org.apache.hadoop.yarn.api.records.NodeId nodeId; RMNode rmNode = null;//from w w w.java2 s . c om if (hopRMNodeComps != null) { nodeId = ConverterUtils.toNodeId(hopRMNodeComps.getRMNodeId()); rmNode = rmContext.getRMNodes().get(nodeId); // The first time we are receiving the RMNode, this will happen when the node registers if (rmNode == null) { // Retrieve heartbeat boolean nextHeartbeat = true; // Create Resource Resource resource = null; if (hopRMNodeComps.getHopResource() != null) { resource = Resource.newInstance(hopRMNodeComps.getHopResource().getMemory(), hopRMNodeComps.getHopResource().getVirtualCores()); } else { LOG.error("ResourceOption should not be null"); resource = Resource.newInstance(0, 0); } /*rmNode = new RMNodeImplDist(nodeId, rmContext, hopRMNodeComps.getHopRMNode().getHostName(), hopRMNodeComps.getHopRMNode().getCommandPort(), hopRMNodeComps.getHopRMNode().getHttpPort(), ResourceTrackerService.resolve(hopRMNodeComps.getHopRMNode().getHostName()), resourceOption, hopRMNodeComps.getHopRMNode().getNodemanagerVersion(), hopRMNodeComps.getHopRMNode().getHealthReport(), hopRMNodeComps.getHopRMNode().getLastHealthReportTime(), nextHeartbeat);*/ rmNode = new RMNodeImplDist(nodeId, rmContext, hopRMNodeComps.getHopRMNode().getHostName(), hopRMNodeComps.getHopRMNode().getCommandPort(), hopRMNodeComps.getHopRMNode().getHttpPort(), ResourceTrackerService.resolve(hopRMNodeComps.getHopRMNode().getHostName()), resource, hopRMNodeComps.getHopRMNode().getNodemanagerVersion()); // Force Java to put the host in cache NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); } // Update the RMNode if (hopRMNodeComps.getHopRMNode() != null) { ((RMNodeImplDist) rmNode).setState(hopRMNodeComps.getHopRMNode().getCurrentState()); } if (hopRMNodeComps.getHopUpdatedContainerInfo() != null) { List<io.hops.metadata.yarn.entity.UpdatedContainerInfo> hopUpdatedContainerInfoList = hopRMNodeComps .getHopUpdatedContainerInfo(); if (hopUpdatedContainerInfoList != null && !hopUpdatedContainerInfoList.isEmpty()) { ConcurrentLinkedQueue<org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> updatedContainerInfoQueue = new ConcurrentLinkedQueue<>(); Map<Integer, org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo> ucis = new HashMap<>(); LOG.debug(hopRMNodeComps.getRMNodeId() + " getting ucis " + hopUpdatedContainerInfoList.size() + " pending event " + hopRMNodeComps.getPendingEvent().getId().getEventId()); for (io.hops.metadata.yarn.entity.UpdatedContainerInfo hopUCI : hopUpdatedContainerInfoList) { if (!ucis.containsKey(hopUCI.getUpdatedContainerInfoId())) { ucis.put(hopUCI.getUpdatedContainerInfoId(), new org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo( new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(), new ArrayList<org.apache.hadoop.yarn.api.records.ContainerStatus>(), hopUCI.getUpdatedContainerInfoId())); } ContainerId cid = ConverterUtils.toContainerId(hopUCI.getContainerId()); io.hops.metadata.yarn.entity.ContainerStatus hopContainerStatus = hopRMNodeComps .getHopContainersStatusMap().get(hopUCI.getContainerId()); org.apache.hadoop.yarn.api.records.ContainerStatus conStatus = org.apache.hadoop.yarn.api.records.ContainerStatus .newInstance(cid, ContainerState.valueOf(hopContainerStatus.getState()), hopContainerStatus.getDiagnostics(), hopContainerStatus.getExitstatus()); // Check ContainerStatus state to add it in the appropriate list if (conStatus != null) { LOG.debug("add uci for container " + conStatus.getContainerId() + " status " + conStatus.getState()); if (conStatus.getState().equals(ContainerState.RUNNING)) { ucis.get(hopUCI.getUpdatedContainerInfoId()).getNewlyLaunchedContainers() .add(conStatus); } else if (conStatus.getState().equals(ContainerState.COMPLETE)) { ucis.get(hopUCI.getUpdatedContainerInfoId()).getCompletedContainers() .add(conStatus); } } } for (org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo uci : ucis .values()) { updatedContainerInfoQueue.add(uci); } ((RMNodeImplDist) rmNode).setUpdatedContainerInfo(updatedContainerInfoQueue); } else { LOG.debug(hopRMNodeComps.getRMNodeId() + " hopUpdatedContainerInfoList = null || hopUpdatedContainerInfoList.isEmpty() " + hopRMNodeComps.getPendingEvent().getId().getEventId()); } } else { LOG.debug(hopRMNodeComps.getRMNodeId() + " hopRMNodeFull.getHopUpdatedContainerInfo()=null " + hopRMNodeComps.getPendingEvent().getId().getEventId()); } } return rmNode; }
From source file:org.apache.myriad.scheduler.fgs.NMHeartBeatHandler.java
License:Apache License
private boolean containerInUse(ContainerStatus status) { return (status.getState() == ContainerState.NEW || status.getState() == ContainerState.RUNNING); }
From source file:org.apache.samza.validation.TestYarnJobValidationTool.java
License:Apache License
@Test public void testValidateContainerCount() throws Exception { List<ContainerReport> containerReports = new ArrayList<>(); for (int i = 0; i <= containerCount; i++) { ContainerReport report = mock(ContainerReport.class); when(report.getContainerState()).thenReturn(ContainerState.RUNNING); containerReports.add(report);// www . j ava2 s .c o m } when(client.getContainers(attemptId)).thenReturn(containerReports); assertTrue(tool.validateContainerCount(attemptId) == (containerCount + 1)); containerReports.remove(0); exception.expect(SamzaException.class); tool.validateContainerCount(attemptId); }
From source file:org.apache.samza.validation.YarnJobValidationTool.java
License:Apache License
public int validateContainerCount(ApplicationAttemptId attemptId) throws Exception { int runningContainerCount = 0; for (ContainerReport containerReport : this.client.getContainers(attemptId)) { if (containerReport.getContainerState() == ContainerState.RUNNING) { ++runningContainerCount;/*from www . ja va2 s .c o m*/ } } // expected containers to be the configured job containers plus the AppMaster container int containerExpected = this.config.getContainerCount() + 1; if (runningContainerCount == containerExpected) { log.info("Container count matches. " + runningContainerCount + " containers are running."); return runningContainerCount; } else { throw new SamzaException("Container count does not match. " + runningContainerCount + " containers are running, while " + containerExpected + " is expected."); } }
From source file:org.apache.tajo.master.YarnContainerProxy.java
License:Apache License
@Override @SuppressWarnings("unchecked") public synchronized void launch(ContainerLaunchContext commonContainerLaunchContext) { LOG.info("Launching Container with Id: " + containerID); if (this.state == ContainerState.KILLED_BEFORE_LAUNCH) { state = ContainerState.DONE;//from www .ja v a 2 s .co m LOG.error("Container (" + containerID + " was killed before it was launched"); return; } ContainerManagementProtocol proxy = null; try { proxy = getCMProxy(containerID, containerMgrAddress, containerToken); // Construct the actual Container ContainerLaunchContext containerLaunchContext = createContainerLaunchContext( commonContainerLaunchContext); // Now launch the actual container List<StartContainerRequest> startRequestList = new ArrayList<StartContainerRequest>(); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequestList.add(startRequest); StartContainersRequest startRequests = Records.newRecord(StartContainersRequest.class); startRequests.setStartContainerRequests(startRequestList); StartContainersResponse response = proxy.startContainers(startRequests); ByteBuffer portInfo = response.getAllServicesMetaData().get(PullServerAuxService.PULLSERVER_SERVICEID); if (portInfo != null) { port = PullServerAuxService.deserializeMetaData(portInfo); } LOG.info("PullServer port returned by ContainerManager for " + containerID + " : " + port); if (port < 0) { this.state = ContainerState.FAILED; throw new IllegalStateException( "Invalid shuffle port number " + port + " returned for " + containerID); } this.state = ContainerState.RUNNING; this.hostName = containerMgrAddress.split(":")[0]; context.getResourceAllocator().addContainer(containerID, this); } catch (Throwable t) { String message = "Container launch failed for " + containerID + " : " + StringUtils.stringifyException(t); this.state = ContainerState.FAILED; LOG.error(message); } finally { if (proxy != null) { yarnRPC.stopProxy(proxy, conf); } } }
From source file:org.springframework.yarn.am.container.DefaultContainerLauncher.java
License:Apache License
/** * Contains the logic to do the actual polling. * * @return True if this poll operation did something, False otherwise *///from w w w. jav a2s. c om private boolean doPoll() { boolean result = false; if (log.isDebugEnabled()) { log.debug("Checking status of containers previousely launched"); } for (Iterator<Container> iterator = launched.iterator(); iterator.hasNext();) { Container container = iterator.next(); ContainerStatus status = getCmTemplate(container).getContainerStatus(); ContainerState state = status.getState(); if (state.equals(ContainerState.COMPLETE)) { iterator.remove(); } else if (state.equals(ContainerState.RUNNING)) { iterator.remove(); if (getYarnEventPublisher() != null) { getYarnEventPublisher().publishContainerLaunched(this, container); } } } return result; }
From source file:uk.ac.gla.terrier.probos.controller.ControllerServer.java
License:Open Source License
@Override public PBSJobStatusLight getJobStatus(int jobId, int requestType) throws Exception { if (requestType > 5 || requestType < 0) throw new IllegalArgumentException("requestType must be [0,1,2,3,4,5]"); char state = '*'; if (!jobArray.containsKey(jobId)) state = '?'; final JobInformation ji = jobArray.get(jobId); final PBSJob job = ji != null ? ji.jobSpec : null; YarnClientService kittenClient = ji != null ? ji.kitten : null; ApplicationReport appReport = null;/*from w ww.ja va 2s.co m*/ if (kittenClient == null || (appReport = kittenClient.getApplicationReport()) == null) { state = '?'; if (jobHolds.get(jobId) != null) { state = 'H'; } } else { YarnApplicationState appState = appReport.getYarnApplicationState(); if (kittenClient.isApplicationFinished()) state = 'E'; else switch (appState) { case NEW: case NEW_SAVING: case ACCEPTED: case SUBMITTED: state = 'Q'; break; case FAILED: case KILLED: case FINISHED: state = 'E'; break; case RUNNING: state = 'R'; break; default: state = '?'; break; } } String timeUse = appReport == null ? "0" : Utils.makeTime(appReport.getApplicationResourceUsageReport().getVcoreSeconds()); String appURL = appReport == null ? "" : appReport.getTrackingUrl(); PBSJobStatusLight rtr = null; String nodes = null; List<ContainerReport> cReports = null; String appId = null; if (requestType == 0) { rtr = new PBSJobStatusLight(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL); } else if (requestType == 4) { checkOwnerOrRoot(ji); JobInteractiveInfo jii = ji != null ? ji.interactive : null; rtr = new PBSJobStatusInteractive(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, jii != null ? jii.hostname : null, jii != null ? jii.port : -1, jii != null ? jii.secret : null); } else if (requestType == 5) { checkOwnerOrRoot(ji); JobDistributedInfo jid = ji != null ? ji.distributed : null; String secret = jid != null ? jid.secret : null; String[] hostnames = jid != null ? jid.hostnames.toArray(new String[0]) : null; int[] ports = jid != null ? jid.ports.toArray() : null; rtr = new PBSJobStatusDistributed(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, hostnames, ports, secret); } //we need the nodes also else if (requestType >= 1) { if (kittenClient != null) { ApplicationId aid = kittenClient.getApplicationReport().getApplicationId(); appId = aid.toString(); List<ApplicationAttemptReport> aaids = yClient.getApplicationAttempts(aid); ApplicationAttemptId aaid = aaids.get(aaids.size() - 1).getApplicationAttemptId(); cReports = yClient.getContainers(aaid); StringBuilder sNodes = new StringBuilder(); if (cReports.size() > 0) { for (ContainerReport cReport : cReports) { if (cReport.getContainerState() == ContainerState.RUNNING) { sNodes.append(cReport.getAssignedNode().getHost()); sNodes.append("+"); } } //remove trailing , sNodes.setLength(sNodes.length() - 1); } nodes = sNodes.toString(); } if (requestType == 1) { rtr = new PBSJobStatusNodes(jobId, job.getArrayTaskIds() != null, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, nodes); } else if (requestType == 2) { String[] tContainers; if (job != null) { tContainers = job.getArrayTaskIds() != null ? ji.array2Container.values(new String[0]) : new String[] { ji.taskContainerId }; } else { tContainers = new String[0]; } String trackingURL = appReport != null ? appReport.getTrackingUrl() : null; rtr = new PBSJobStatusFat(jobId, job != null ? job.getArrayTaskIds() != null : false, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, nodes, ji != null ? ji.jobSpec : null, ji != null ? ji.masterContainerId : null, tContainers, trackingURL, appId); } else if (requestType == 3) { int[] arrayIds = job != null ? JobUtils.getTaskArrayItems(job.getArrayTaskIds()) : new int[0]; if (arrayIds == null) arrayIds = new int[0]; char[] states = new char[arrayIds.length]; //String[] walltime = new String[arrayIds.length]; int i = 0; for (int arid : arrayIds) { String containerStatus = ji.array2Container.get(arid); if (containerStatus == null) states[i] = 'Q'; else if (containerStatus.equals("DONE")) states[i] = 'C'; else if (containerStatus.equals("ABORTED")) states[i] = 'C'; else { states[i] = 'R'; // ContainerId c = ContainerId.fromString(containerStatus); // for(ContainerReport cReport : cReports) // { // if (cReport.getContainerId().equals(c) // { // walltime[i] = cReport. // } } i++; } rtr = new PBSJobArrayStatusLight(jobId, job != null ? job.getJob_Name() : null, job != null ? job.getJob_Owner() : null, timeUse, state, job != null ? job.getQueue() : null, appURL, arrayIds, states); } else { //this should not be reached. throw new IllegalArgumentException("Bad requestType"); } } return rtr; }