List of usage examples for org.apache.hadoop.yarn.api.protocolrecords StartContainerRequest setContainerLaunchContext
@Public @Stable public abstract void setContainerLaunchContext(ContainerLaunchContext context);
ContainerLaunchContext for the container to be started by the NodeManager From source file:com.continuuity.weave.internal.yarn.Hadoop20YarnNMClient.java
License:Apache License
@Override public Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext) { ContainerLaunchContext context = launchContext.getLaunchContext(); context.setUser(System.getProperty("user.name")); Container container = containerInfo.getContainer(); context.setContainerId(container.getId()); context.setResource(container.getResource()); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(context); ContainerManager manager = connectContainerManager(container); try {/*from w ww. j a va 2 s. com*/ manager.startContainer(startRequest); return new ContainerTerminator(container, manager); } catch (YarnRemoteException e) { LOG.error("Error in launching process", e); throw Throwables.propagate(e); } }
From source file:org.apache.hama.bsp.BSPTaskLauncher.java
License:Apache License
private GetContainerStatusesRequest setupContainer(Container allocatedContainer, ContainerManagementProtocol cm, String user, int id) throws IOException, YarnException { LOG.info("Setting up a container for user " + user + " with id of " + id + " and containerID of " + allocatedContainer.getId() + " as " + user); // Now we setup a ContainerLaunchContext ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LocalResource packageResource = Records.newRecord(LocalResource.class); FileSystem fs = FileSystem.get(conf); Path packageFile = new Path(System.getenv(YARNBSPConstants.HAMA_YARN_LOCATION)); URL packageUrl = ConverterUtils .getYarnUrlFromPath(packageFile.makeQualified(fs.getUri(), fs.getWorkingDirectory())); LOG.info("PackageURL has been composed to " + packageUrl.toString()); try {// ww w .j a v a2 s .co m LOG.info("Reverting packageURL to path: " + ConverterUtils.getPathFromYarnURL(packageUrl)); } catch (URISyntaxException e) { LOG.fatal("If you see this error the workarround does not work", e); } packageResource.setResource(packageUrl); packageResource.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_SIZE))); packageResource.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_YARN_TIMESTAMP))); packageResource.setType(LocalResourceType.FILE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(YARNBSPConstants.APP_MASTER_JAR_PATH, packageResource); Path hamaReleaseFile = new Path(System.getenv(YARNBSPConstants.HAMA_RELEASE_LOCATION)); URL hamaReleaseUrl = ConverterUtils .getYarnUrlFromPath(hamaReleaseFile.makeQualified(fs.getUri(), fs.getWorkingDirectory())); LOG.info("Hama release URL has been composed to " + hamaReleaseUrl.toString()); LocalResource hamaReleaseRsrc = Records.newRecord(LocalResource.class); hamaReleaseRsrc.setResource(hamaReleaseUrl); hamaReleaseRsrc.setSize(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_SIZE))); hamaReleaseRsrc.setTimestamp(Long.parseLong(System.getenv(YARNBSPConstants.HAMA_RELEASE_TIMESTAMP))); hamaReleaseRsrc.setType(LocalResourceType.ARCHIVE); hamaReleaseRsrc.setVisibility(LocalResourceVisibility.APPLICATION); localResources.put(YARNBSPConstants.HAMA_SYMLINK, hamaReleaseRsrc); ctx.setLocalResources(localResources); /* * TODO Package classpath seems not to work if you're in pseudo distributed * mode, because the resource must not be moved, it will never be unpacked. * So we will check if our jar file has the file:// prefix and put it into * the CP directly */ StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$()) .append(File.pathSeparatorChar).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } classPathEnv.append(File.pathSeparator); classPathEnv .append("./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/*"); classPathEnv.append(File.pathSeparator); classPathEnv.append( "./" + YARNBSPConstants.HAMA_SYMLINK + "/" + YARNBSPConstants.HAMA_RELEASE_VERSION + "/lib/*"); Vector<CharSequence> vargs = new Vector<CharSequence>(); vargs.add("${JAVA_HOME}/bin/java"); vargs.add("-cp " + classPathEnv + ""); vargs.add(BSPRunner.class.getCanonicalName()); vargs.add(jobId.getJtIdentifier()); vargs.add(Integer.toString(id)); vargs.add(this.jobFile.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/bsp.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); ctx.setCommands(commands); LOG.info("Starting command: " + commands); StartContainerRequest startReq = Records.newRecord(StartContainerRequest.class); startReq.setContainerLaunchContext(ctx); startReq.setContainerToken(allocatedContainer.getContainerToken()); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(startReq); StartContainersRequest requestList = StartContainersRequest.newInstance(list); cm.startContainers(requestList); GetContainerStatusesRequest statusReq = Records.newRecord(GetContainerStatusesRequest.class); List<ContainerId> containerIds = new ArrayList<ContainerId>(); containerIds.add(allocatedContainer.getId()); statusReq.setContainerIds(containerIds); return statusReq; }
From source file:org.apache.samza.job.yarn.ContainerUtil.java
License:Apache License
protected void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus;//w w w . j a va2 s. com try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaException("IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { ioe.printStackTrace(); throw new SamzaException("IO Exception when writing credentials to output buffer"); } ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(Collections.singletonMap("__package", packageResource)); log.debug("setting package to {}", packageResource); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaException("Received YarnException when starting container: " + container.getId()); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaException("Received IOException when starting container: " + container.getId()); } }
From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath./*from w w w .j av a 2 s .c om*/ */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws SamzaContainerLaunchException { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaContainerLaunchException( "IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { log.error("IOException when writing credentials.", ioe); throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer"); } ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(Collections.singletonMap("__package", packageResource)); log.debug("setting package to {}", packageResource); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaContainerLaunchException( "Received YarnException when starting container: " + container.getId(), ye); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaContainerLaunchException( "Received IOException when starting container: " + container.getId(), ioe); } }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath./* ww w . j ava 2 s . co m*/ */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws IOException { LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); packageResource.setResource(packageUrl); log.debug("Set package resource in YarnContainerRunner for {}", packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens to start the container Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((org.apache.hadoop.security.token.Token) iter.next()).decodeIdentifier(); if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); Map<String, LocalResource> localResourceMap = new HashMap<>(); localResourceMap.put("__package", packageResource); // include the resources from the universal resource configurations LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration); localResourceMap.putAll(resourceMapper.getResourceMap()); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(localResourceMap); if (UserGroupInformation.isSecurityEnabled()) { Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls(); if (!acls.isEmpty()) { context.setApplicationACLs(acls); } } log.debug("Setting localResourceMap to {}", localResourceMap); log.debug("Setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); log.info( "Making an async start request for Container ID: {} on host: {} with local resource map: {} and context: {}", container.getId(), container.getNodeHttpAddress(), localResourceMap.toString(), context); nmClientAsync.startContainerAsync(container, context); }
From source file:org.apache.samza.job.yarn.YarnContainerRunner.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath.// w w w.j av a2s. co m */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws SamzaContainerLaunchException { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // TODO: SAMZA-1144 remove the customized approach for package resource and use the common one. // But keep it now for backward compatibility. // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaContainerLaunchException( "IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); log.info("set package Resource in YarnContainerRunner for {}", packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { log.error("IOException when writing credentials.", ioe); throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer"); } Map<String, LocalResource> localResourceMap = new HashMap<>(); localResourceMap.put("__package", packageResource); // include the resources from the universal resource configurations LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration); localResourceMap.putAll(resourceMapper.getResourceMap()); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(localResourceMap); log.debug("setting localResourceMap to {}", localResourceMap); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaContainerLaunchException( "Received YarnException when starting container: " + container.getId(), ye); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaContainerLaunchException( "Received IOException when starting container: " + container.getId(), ioe); } }
From source file:org.apache.tajo.master.YarnContainerProxy.java
License:Apache License
@Override @SuppressWarnings("unchecked") public synchronized void launch(ContainerLaunchContext commonContainerLaunchContext) { LOG.info("Launching Container with Id: " + containerID); if (this.state == ContainerState.KILLED_BEFORE_LAUNCH) { state = ContainerState.DONE;//from w ww . j a v a2 s .c o m LOG.error("Container (" + containerID + " was killed before it was launched"); return; } ContainerManagementProtocol proxy = null; try { proxy = getCMProxy(containerID, containerMgrAddress, containerToken); // Construct the actual Container ContainerLaunchContext containerLaunchContext = createContainerLaunchContext( commonContainerLaunchContext); // Now launch the actual container List<StartContainerRequest> startRequestList = new ArrayList<StartContainerRequest>(); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(containerLaunchContext); startRequestList.add(startRequest); StartContainersRequest startRequests = Records.newRecord(StartContainersRequest.class); startRequests.setStartContainerRequests(startRequestList); StartContainersResponse response = proxy.startContainers(startRequests); ByteBuffer portInfo = response.getAllServicesMetaData().get(PullServerAuxService.PULLSERVER_SERVICEID); if (portInfo != null) { port = PullServerAuxService.deserializeMetaData(portInfo); } LOG.info("PullServer port returned by ContainerManager for " + containerID + " : " + port); if (port < 0) { this.state = ContainerState.FAILED; throw new IllegalStateException( "Invalid shuffle port number " + port + " returned for " + containerID); } this.state = ContainerState.RUNNING; this.hostName = containerMgrAddress.split(":")[0]; context.getResourceAllocator().addContainer(containerID, this); } catch (Throwable t) { String message = "Container launch failed for " + containerID + " : " + StringUtils.stringifyException(t); this.state = ContainerState.FAILED; LOG.error(message); } finally { if (proxy != null) { yarnRPC.stopProxy(proxy, conf); } } }
From source file:org.deeplearning4j.iterativereduce.runtime.yarn.ContainerManagerHandler.java
License:Apache License
public StartContainerResponse startContainer(List<String> commands, Map<String, LocalResource> localResources, Map<String, String> env) throws IOException { if (containerManager == null) throw new IllegalStateException("Cannot start a continer before connecting to the container manager!"); ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); ctx.setContainerId(container.getId()); ctx.setResource(container.getResource()); ctx.setLocalResources(localResources); ctx.setCommands(commands);//from ww w .j a va 2 s. c o m ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); ctx.setEnvironment(env); if (LOG.isDebugEnabled()) { LOG.debug("Using ContainerLaunchContext with" + ", containerId=" + ctx.getContainerId() + ", memory=" + ctx.getResource().getMemory() + ", localResources=" + ctx.getLocalResources().toString() + ", commands=" + ctx.getCommands().toString() + ", env=" + ctx.getEnvironment().toString()); } StartContainerRequest request = Records.newRecord(StartContainerRequest.class); request.setContainerLaunchContext(ctx); LOG.info("Starting container, containerId=" + container.getId().toString() + ", host=" + container.getNodeId().getHost() + ", http=" + container.getNodeHttpAddress()); return containerManager.startContainer(request); }
From source file:org.springframework.yarn.am.container.DefaultContainerLauncher.java
License:Apache License
@Override public void launchContainer(Container container, List<String> commands) { if (log.isDebugEnabled()) { log.debug("Launching container: " + container); }/*from www .j a v a 2 s. c o m*/ ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); String stagingId = container.getId().getApplicationAttemptId().getApplicationId().toString(); getResourceLocalizer().setStagingId(stagingId); ctx.setLocalResources(getResourceLocalizer().getResources()); ctx.setCommands(commands); // Yarn doesn't tell container what is its container id // so we do it here Map<String, String> env = getEnvironment(); env.put(YarnSystemConstants.SYARN_CONTAINER_ID, ConverterUtils.toString(container.getId())); ctx.setEnvironment(env); ctx = getInterceptors().preLaunch(container, ctx); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(ctx); startContainerRequest.setContainerToken(container.getContainerToken()); StartContainersRequest startContainersRequest = Records.newRecord(StartContainersRequest.class); ArrayList<StartContainerRequest> startContainerRequestList = new ArrayList<StartContainerRequest>(); startContainerRequestList.add(startContainerRequest); startContainersRequest.setStartContainerRequests(startContainerRequestList); StartContainersResponse startContainersResponse = getCmTemplate(container) .startContainers(startContainersRequest); Map<ContainerId, SerializedException> failedRequests = startContainersResponse.getFailedRequests(); List<ContainerId> successfullyStartedContainers = startContainersResponse .getSuccessfullyStartedContainers(); // TODO: handle failed/success // notify interested parties of new launched container if (getYarnEventPublisher() != null) { getYarnEventPublisher().publishContainerLaunched(this, container); } }