List of usage examples for org.apache.hadoop.yarn.api.records ContainerLaunchContext setCommands
@Public @Stable public abstract void setCommands(List<String> commands);
From source file:yarnkit.container.ContainerLaunchContextFactory.java
License:Apache License
public ContainerLaunchContext duplicate(@CheckForNull ContainerLaunchContext original) { Preconditions.checkNotNull(original, "ContainerLaunchContext should not be null"); ContainerLaunchContext copy = Records.newRecord(ContainerLaunchContext.class); copy.setCommands(original.getCommands()); copy.setEnvironment(original.getEnvironment()); copy.setLocalResources(original.getLocalResources()); ByteBuffer token = original.getTokens(); if (token != null) { copy.setTokens(token.duplicate()); }/* w w w. j a va 2 s . c o m*/ return copy; }
From source file:yrun.YarnRunner.java
License:Apache License
public void execute() throws IOException, YarnException, InterruptedException { LOG.info("Using application path [" + _installPath + "]"); Path jarPath = installThisJar(_installPath, _appJarFile); LOG.info("Driver installed [" + jarPath + "]"); List<Path> installedArchivePathList = install(_installPath, _archivePathList); for (Path p : installedArchivePathList) { LOG.info("Archive installed [" + p + "]"); }//from w w w . j a va2 s . c o m YarnRunnerArgs yarnRunnerArgs = new YarnRunnerArgs(); yarnRunnerArgs.setCommand(_command); Path argsPath = installThisArgs(_installPath, yarnRunnerArgs); final YarnClient client = YarnClient.createYarnClient(); _configuration.setInt("yarn.nodemanager.delete.debug-delay-sec", (int) TimeUnit.HOURS.toSeconds(1)); client.init(_configuration); client.start(); YarnClientApplication app = client.createApplication(); ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv, _appJarFile); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); { LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); localResources.put(jarPath.getName(), appMasterJar); } { LocalResource appMasterArgs = Records.newRecord(LocalResource.class); setupAppMasterArgs(argsPath, appMasterArgs); localResources.put(MASTER_JSON, appMasterArgs); } List<String> vargs = new ArrayList<String>(); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); vargs.add("-Xmx256m"); vargs.add("-Djava.net.preferIPv4Stack=true"); vargs.add(YarnRunnerApplicationMaster.class.getName()); String strCommand = "(echo ENV && set && echo CURRENT_DIR_LISTING && ls -la && echo PWD && pwd && (" + StringUtils.join(" ", vargs) + "))"; strCommand += " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"; strCommand += " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"; LOG.debug("Application Master command [" + strCommand + "]"); amContainer.setCommands(Collections.singletonList(strCommand)); amContainer.setLocalResources(localResources); amContainer.setEnvironment(appMasterEnv); Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(_yarnName); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); if (_queue != null) { appContext.setQueue(_queue); } appContext.setApplicationType("yrun"); ApplicationId appId = appContext.getApplicationId(); AtomicBoolean shutdown = new AtomicBoolean(); if (!_isDaemon) { addShutdownHook(client, appId, shutdown); } LOG.info("Submitting application with id [" + appId + "]"); client.submitApplication(appContext); ApplicationReport report; YarnApplicationState state; do { report = client.getApplicationReport(appId); state = report.getYarnApplicationState(); if (state == YarnApplicationState.RUNNING) { if (_isDaemon) { LOG.info("Application is running. This is a daemon application driver program exiting."); return; } } Thread.sleep(100); } while (isNoLongerRunning(state)); shutdown.set(true); LOG.info("Application has finished with state [" + state + "]"); }
From source file:yrun.YarnRunnerApplicationMaster.java
License:Apache License
public void run(String[] args) throws Exception { JsonParser parser = new JsonParser(); JsonElement element = parser.parse(new FileReader(YarnRunner.MASTER_JSON)); LOG.info("Json [" + element + "]"); JsonObject jsonObject = (JsonObject) element; int priority = jsonObject.get("priority").getAsInt(); int numberOfContainers = jsonObject.get("numberOfContainers").getAsInt(); int memory = jsonObject.get("memory").getAsInt(); int vCores = jsonObject.get("vCores").getAsInt(); String command = jsonObject.get("command").getAsString(); // startHttpServer(); // InetSocketAddress address = _server.getAddress(); // LOG.info("Http server started at [" + address + "]"); // String appHostName = "app-host-name"; // int appHostPort = address.getPort(); // String appTrackingUrl = "http://" + address.getHostName() + ":" + // appHostPort + "/"; // LOG.info("App Tracking Url [" + appTrackingUrl + "]"); // Initialize clients to ResourceManager and NodeManagers Configuration conf = new YarnConfiguration(); AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient(); rmClient.init(conf);/*from w w w . j ava 2s.com*/ rmClient.start(); NMClient nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with ResourceManager LOG.info("Register Application Master 0"); String appHostName = ""; int appHostPort = 0; String appTrackingUrl = ""; rmClient.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl); LOG.info("Register Application Master 1"); // Priority for worker containers - priorities are intra-application Priority priorityRecord = Records.newRecord(Priority.class); priorityRecord.setPriority(priority); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); capability.setVirtualCores(vCores); // Make container requests to ResourceManager for (int i = 0; i < numberOfContainers; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priorityRecord); LOG.info("Making resource request for [" + i + "]"); rmClient.addContainerRequest(containerAsk); } // Obtain allocated containers, launch and check for responses int responseId = 0; int completedContainers = 0; long lastReport = 0; List<Container> containers = new ArrayList<Container>(); while (completedContainers < numberOfContainers) { if (lastReport + TimeUnit.SECONDS.toNanos(5) < System.nanoTime()) { for (Container container : containers) { ContainerId containerId = container.getId(); NodeId nodeId = container.getNodeId(); ContainerStatus containerStatus = nmClient.getContainerStatus(containerId, nodeId); LOG.info("NodeId [" + nodeId + "] Container Status [" + containerStatus + "]"); // Figure out } lastReport = System.nanoTime(); } AllocateResponse response = rmClient.allocate(responseId++); for (Container container : response.getAllocatedContainers()) { containers.add(container); // Launch container by create ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setCommands( Collections.singletonList(command + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout2" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr")); LOG.info("Launching container " + container.getId()); nmClient.startContainer(container, ctx); } for (ContainerStatus status : response.getCompletedContainersStatuses()) { completedContainers++; LOG.info("Completed container " + status.getContainerId()); } Thread.sleep(100); } // _server.stop(0); // Un-register with ResourceManager rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); }