List of usage examples for org.apache.hadoop.yarn.api.records ContainerStatus getContainerId
@Public @Stable public abstract ContainerId getContainerId();
ContainerId of the container. From source file:org.springframework.yarn.am.monitor.DefaultContainerMonitorTests.java
License:Apache License
/** * Mock {@link ContainerStatus}//from w w w . j a v a 2 s. c om * * @param containerId the {@link ContainerId} * @param containerState the {@link ContainerState} * @param exitStatus the container exit status * @return mocked {@link ContainerStatus} */ public static ContainerStatus getMockContainerStatus(ContainerId containerId, ContainerState containerState, int exitStatus) { ContainerStatus status = mock(ContainerStatus.class); when(status.getContainerId()).thenReturn(containerId); when(status.getState()).thenReturn(containerState); when(status.getExitStatus()).thenReturn(exitStatus); return status; }
From source file:org.springframework.yarn.batch.am.AbstractBatchAppmaster.java
License:Apache License
@Override protected void onContainerCompleted(ContainerStatus status) { super.onContainerCompleted(status); // find assigned container for step execution ContainerId containerId = status.getContainerId(); StepExecution stepExecution = containerToStepMap.get(containerId); if (stepExecution != null) { for (Entry<StepExecution, Set<StepExecution>> entry : masterExecutions.entrySet()) { Set<StepExecution> set = entry.getValue(); if (set.remove(stepExecution)) { if (log.isDebugEnabled()) { log.debug("stepExecution=" + stepExecution + " removed"); }/*from w w w . ja va 2 s . co m*/ // modified, but it back masterExecutions.put(entry.getKey(), set); } if (set.size() == 0) { // we consumed all executions, send complete event // TODO: we could track failures getYarnEventPublisher().publishEvent(new PartitionedStepExecutionEvent(this, entry.getKey())); stepExecutionStateListener.state(PartitionedStepExecutionState.COMPLETED, entry.getKey()); } } } else { log.warn("No assigned step execution for containerId=" + containerId); } // finally notify allocator for release getAllocator().releaseContainer(containerId); }
From source file:org.springframework.yarn.boot.MockUtils.java
License:Apache License
public static ContainerStatus getMockContainerStatus(ContainerId containerId, ContainerState containerState, int exitStatus) { ContainerStatus status = mock(ContainerStatus.class); when(status.getContainerId()).thenReturn(containerId); when(status.getState()).thenReturn(containerState); when(status.getExitStatus()).thenReturn(exitStatus); return status; }
From source file:org.springframework.yarn.examples.AbstractManagedContainerGroupsAppmaster.java
License:Apache License
@Override protected void onContainerCompleted(ContainerStatus status) { log.info("XXX onContainerCompleted: " + status); super.onContainerCompleted(status); getMonitor().monitorContainer(status); int exitStatus = status.getExitStatus(); ContainerId containerId = status.getContainerId(); boolean handled = false; if (exitStatus > 0) { handled = onContainerFailed(containerId); if (!handled) { setFinalApplicationStatus(FinalApplicationStatus.FAILED); notifyCompleted();/*from w ww . ja v a 2s .c om*/ } } else { if (isComplete()) { notifyCompleted(); } } }
From source file:org.springframework.yarn.examples.CustomAppmaster.java
License:Apache License
@Override protected boolean onContainerFailed(ContainerStatus status) { ContainerId containerId = status.getContainerId(); log.debug("onContainerFailed status: " + status); log.debug("onContainerFailed containerId: " + containerId); if (status.getExitStatus() > 0) { failed.add(containerId);/*from w w w . ja v a2s . c o m*/ getAllocator().allocateContainers(1); } return true; }
From source file:runtime.starter.MPJAppMaster.java
License:Open Source License
public void run() throws Exception { try {/*www . j av a 2 s.c o m*/ appMasterSock = new Socket(serverName, ioServerPort); //redirecting stdout and stderr System.setOut(new PrintStream(appMasterSock.getOutputStream(), true)); System.setErr(new PrintStream(appMasterSock.getOutputStream(), true)); } catch (Exception exp) { exp.printStackTrace(); } FileSystem fs = FileSystem.get(conf); Path wrapperDest = new Path(wrapperPath); FileStatus destStatus = fs.getFileStatus(wrapperDest); Path userFileDest = new Path(userJarPath); FileStatus destStatusClass = fs.getFileStatus(userFileDest); // Initialize AM <--> RM communication protocol AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf); rmClient.start(); // Initialize AM <--> NM communication protocol NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager RegisterApplicationMasterResponse registerResponse = rmClient.registerApplicationMaster("", 0, ""); // Priority for containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(mpjContainerPriority); maxMem = registerResponse.getMaximumResourceCapability().getMemory(); if (debugYarn) { System.out.println("[MPJAppMaster]: Max memory capability resources " + "in cluster: " + maxMem); } if (containerMem > maxMem) { System.out.println("[MPJAppMaster]: container memory specified above " + "threshold of cluster! Using maximum memory for " + "containers: " + containerMem); containerMem = maxMem; } maxCores = registerResponse.getMaximumResourceCapability().getVirtualCores(); if (debugYarn) { System.out.println("[MPJAppMaster]: Max v-cores capability resources " + "in cluster: " + maxCores); } if (containerCores > maxCores) { System.out.println("[MPJAppMaster]: virtual cores specified above " + "threshold of cluster! Using maximum v-cores for " + "containers: " + containerCores); containerCores = maxCores; } // Resource requirements for containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMem); capability.setVirtualCores(containerCores); // Make container requests to ResourceManager for (int i = 0; i < np; ++i) { ContainerRequest containerReq = new ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerReq); } Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); // Creating Local Resource for Wrapper LocalResource wrapperJar = Records.newRecord(LocalResource.class); wrapperJar.setResource(ConverterUtils.getYarnUrlFromPath(wrapperDest)); wrapperJar.setSize(destStatus.getLen()); wrapperJar.setTimestamp(destStatus.getModificationTime()); wrapperJar.setType(LocalResourceType.ARCHIVE); wrapperJar.setVisibility(LocalResourceVisibility.APPLICATION); // Creating Local Resource for UserClass LocalResource userClass = Records.newRecord(LocalResource.class); userClass.setResource(ConverterUtils.getYarnUrlFromPath(userFileDest)); userClass.setSize(destStatusClass.getLen()); userClass.setTimestamp(destStatusClass.getModificationTime()); userClass.setType(LocalResourceType.ARCHIVE); userClass.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put("mpj-yarn-wrapper.jar", wrapperJar); localResources.put("user-code.jar", userClass); while (allocatedContainers < np) { AllocateResponse response = rmClient.allocate(0); mpiContainers.addAll(response.getAllocatedContainers()); allocatedContainers = mpiContainers.size(); if (allocatedContainers != np) { Thread.sleep(100); } } if (debugYarn) { System.out.println("[MPJAppMaster]: launching " + allocatedContainers + " containers"); } for (Container container : mpiContainers) { ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); List<String> commands = new ArrayList<String>(); commands.add(" $JAVA_HOME/bin/java"); commands.add(" -Xmx" + containerMem + "m"); commands.add(" runtime.starter.MPJYarnWrapper"); commands.add("--serverName"); commands.add(serverName); // server name commands.add("--ioServerPort"); commands.add(Integer.toString(ioServerPort)); // IO server port commands.add("--deviceName"); commands.add(deviceName); // device name commands.add("--className"); commands.add(className); // class name commands.add("--psl"); commands.add(psl); // protocol switch limit commands.add("--np"); commands.add(Integer.toString(np)); // no. of containers commands.add("--rank"); commands.add(" " + Integer.toString(rank++)); // rank //temp sock port to share rank and ports commands.add("--wireUpPort"); commands.add(wireUpPort); if (appArgs != null) { commands.add("--appArgs"); for (int i = 0; i < appArgs.length; i++) { commands.add(appArgs[i]); } } ctx.setCommands(commands); // Set local resource for containers ctx.setLocalResources(localResources); // Set environment for container Map<String, String> containerEnv = new HashMap<String, String>(); setupEnv(containerEnv); ctx.setEnvironment(containerEnv); // Time to start the container nmClient.startContainer(container, ctx); } while (completedContainers < np) { // argument to allocate() is the progress indicator AllocateResponse response = rmClient.allocate(completedContainers / np); for (ContainerStatus status : response.getCompletedContainersStatuses()) { if (debugYarn) { System.out.println("\n[MPJAppMaster]: Container Id - " + status.getContainerId()); System.out.println("[MPJAppMaster]: Container State - " + status.getState().toString()); System.out.println("[MPJAppMaster]: Container Diagnostics - " + status.getDiagnostics()); } ++completedContainers; } if (completedContainers != np) { Thread.sleep(100); } ; } // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); //shutDown AppMaster IO System.out.println("EXIT"); }
From source file:tachyon.yarn.ApplicationMaster.java
License:Apache License
@Override public void onContainersCompleted(List<ContainerStatus> statuses) { for (ContainerStatus status : statuses) { LOG.error("Completed container " + status.getContainerId()); }//from www.j a v a2 s.com }
From source file:uk.ac.gla.terrier.probos.master.ProbosApplicationMasterServiceImpl.java
License:Open Source License
@Override public void onContainersCompleted(List<ContainerStatus> containerList) { super.onContainersCompleted(containerList); //?*HACK*?/* w w w . ja v a 2 s . c om*/ //for some reason, we don't observe onContainerStopped() being called //until the end of the application, but onContainersCompleted() being //called as expected. Hence, we map these through to help the Controller //observe array task completions for (ContainerStatus status : containerList) { ContainerId cid = status.getContainerId(); for (ContainerTracker tracker : trackers) { if (tracker.owns(cid)) if (tracker instanceof ProbosContainerTracker) ((ProbosContainerTracker) tracker).onContainerStopped(status); else tracker.onContainerStopped(cid); } } }
From source file:yarnkit.utils.YarnUtils.java
License:Apache License
public static String getContainerExitStatusMessage(@Nonnull ContainerStatus status) { String containerId = status.getContainerId().toString(); final String msg; final int exitStatus = status.getExitStatus(); switch (exitStatus) { case ContainerExitStatus.SUCCESS: { msg = String.format("Container %s finished succesfully", containerId); break;// w w w .j ava2 s. com } case ContainerExitStatus.ABORTED: { msg = String.format("Container %s aborted", containerId); break; } case ContainerExitStatus.DISKS_FAILED: { msg = String.format("Container %s ran out of disk", containerId); break; } case ContainerExitStatus.PREEMPTED: { msg = String.format("Container %s preempted", containerId); break; } case ContainerExitStatus.INVALID: default: { msg = String.format("Container %s exited with an invalid/unknown exit code: %d", containerId, exitStatus); break; } } String diagnostics = status.getDiagnostics(); if (exitStatus == ContainerExitStatus.SUCCESS || Strings.isNullOrEmpty(diagnostics)) { return msg; } else { return msg + "\nDiagnostics: " + diagnostics; } }
From source file:yrun.YarnRunnerApplicationMaster.java
License:Apache License
public void run(String[] args) throws Exception { JsonParser parser = new JsonParser(); JsonElement element = parser.parse(new FileReader(YarnRunner.MASTER_JSON)); LOG.info("Json [" + element + "]"); JsonObject jsonObject = (JsonObject) element; int priority = jsonObject.get("priority").getAsInt(); int numberOfContainers = jsonObject.get("numberOfContainers").getAsInt(); int memory = jsonObject.get("memory").getAsInt(); int vCores = jsonObject.get("vCores").getAsInt(); String command = jsonObject.get("command").getAsString(); // startHttpServer(); // InetSocketAddress address = _server.getAddress(); // LOG.info("Http server started at [" + address + "]"); // String appHostName = "app-host-name"; // int appHostPort = address.getPort(); // String appTrackingUrl = "http://" + address.getHostName() + ":" + // appHostPort + "/"; // LOG.info("App Tracking Url [" + appTrackingUrl + "]"); // Initialize clients to ResourceManager and NodeManagers Configuration conf = new YarnConfiguration(); AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf);// ww w .j a v a 2 s . co m rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager LOG.info("Register Application Master 0"); String appHostName = ""; int appHostPort = 0; String appTrackingUrl = ""; rmClient.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl); LOG.info("Register Application Master 1"); // Priority for worker containers - priorities are intra-application Priority priorityRecord = Records.newRecord(Priority.class); priorityRecord.setPriority(priority); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); capability.setVirtualCores(vCores); // Make container requests to ResourceManager for (int i = 0; i < numberOfContainers; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priorityRecord); LOG.info("Making resource request for [" + i + "]"); rmClient.addContainerRequest(containerAsk); } // Obtain allocated containers, launch and check for responses int responseId = 0; int completedContainers = 0; long lastReport = 0; List<Container> containers = new ArrayList<Container>(); while (completedContainers < numberOfContainers) { if (lastReport + TimeUnit.SECONDS.toNanos(5) < System.nanoTime()) { for (Container container : containers) { ContainerId containerId = container.getId(); NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = nmClient.getContainerStatus(containerId, nodeId); LOG.info("NodeId [" + nodeId + "] Container Status [" + containerStatus + "]"); // Figure out } lastReport = System.nanoTime(); } AllocateResponse response = rmClient.allocate(responseId++); for (Container container : response.getAllocatedContainers()) { containers.add(container); // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setCommands( Collections.singletonList(command + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout2" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); LOG.info("Launching container " + container.getId()); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { completedContainers++; LOG.info("Completed container " + status.getContainerId()); } Thread.sleep(100); } // _server.stop(0); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }