List of usage examples for org.apache.hadoop.yarn.api.records ContainerLaunchContext setTokens
@Public @Stable public abstract void setTokens(ByteBuffer tokens);
From source file:org.apache.reef.runtime.yarn.driver.unmanaged.UnmanagedAmTest.java
License:Apache License
@Test public void testAmShutdown() throws IOException, YarnException { Assume.assumeTrue("This test requires a YARN Resource Manager to connect to", Boolean.parseBoolean(System.getenv("REEF_TEST_YARN"))); final YarnConfiguration yarnConfig = new YarnConfiguration(); // Start YARN client and register the application final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfig);/*from www .j a v a2 s . com*/ yarnClient.start(); final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class); containerContext.setCommands(Collections.<String>emptyList()); containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap()); containerContext.setEnvironment(Collections.<String, String>emptyMap()); containerContext.setTokens(getTokens()); final ApplicationSubmissionContext appContext = yarnClient.createApplication() .getApplicationSubmissionContext(); appContext.setApplicationName("REEF_Unmanaged_AM_Test"); appContext.setAMContainerSpec(containerContext); appContext.setUnmanagedAM(true); appContext.setQueue("default"); final ApplicationId applicationId = appContext.getApplicationId(); LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId); yarnClient.submitApplication(appContext); LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId); addToken(yarnClient.getAMRMToken(applicationId)); // Start the AM final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this); rmClient.init(yarnConfig); rmClient.start(); final NMClientAsync nmClient = new NMClientAsyncImpl(this); nmClient.init(yarnConfig); nmClient.start(); final RegisterApplicationMasterResponse registration = rmClient .registerApplicationMaster(NetUtils.getHostname(), -1, null); LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null); LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState()); // Shutdown the AM rmClient.stop(); nmClient.stop(); // Get the final application report final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); final YarnApplicationState appState = appReport.getYarnApplicationState(); final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus(); LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus }); Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState); Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus); // Shutdown YARN client yarnClient.stop(); }
From source file:org.apache.samza.job.yarn.ContainerUtil.java
License:Apache License
protected void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus;//w ww . j ava 2 s . c o m try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaException("IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { ioe.printStackTrace(); throw new SamzaException("IO Exception when writing credentials to output buffer"); } ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(Collections.singletonMap("__package", packageResource)); log.debug("setting package to {}", packageResource); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaException("Received YarnException when starting container: " + container.getId()); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaException("Received IOException when starting container: " + container.getId()); } }
From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath.//from w w w .j a v a 2s . c o m */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws SamzaContainerLaunchException { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaContainerLaunchException( "IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { log.error("IOException when writing credentials.", ioe); throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer"); } ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(Collections.singletonMap("__package", packageResource)); log.debug("setting package to {}", packageResource); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaContainerLaunchException( "Received YarnException when starting container: " + container.getId(), ye); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaContainerLaunchException( "Received IOException when starting container: " + container.getId(), ioe); } }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath.//from www.j a va2s . c o m */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws IOException { LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); packageResource.setResource(packageUrl); log.debug("Set package resource in YarnContainerRunner for {}", packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens to start the container Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((org.apache.hadoop.security.token.Token) iter.next()).decodeIdentifier(); if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); Map<String, LocalResource> localResourceMap = new HashMap<>(); localResourceMap.put("__package", packageResource); // include the resources from the universal resource configurations LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration); localResourceMap.putAll(resourceMapper.getResourceMap()); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(localResourceMap); if (UserGroupInformation.isSecurityEnabled()) { Map<ApplicationAccessType, String> acls = yarnConfig.getYarnApplicationAcls(); if (!acls.isEmpty()) { context.setApplicationACLs(acls); } } log.debug("Setting localResourceMap to {}", localResourceMap); log.debug("Setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); log.info( "Making an async start request for Container ID: {} on host: {} with local resource map: {} and context: {}", container.getId(), container.getNodeHttpAddress(), localResourceMap.toString(), context); nmClientAsync.startContainerAsync(container, context); }
From source file:org.apache.samza.job.yarn.YarnContainerRunner.java
License:Apache License
/** * Runs a command as a process on the container. All binaries needed by the physical process are packaged in the URL * specified by packagePath./* ww w . j ava 2 s . co m*/ */ private void startContainer(Path packagePath, Container container, Map<String, String> env, final String cmd) throws SamzaContainerLaunchException { log.info("starting container {} {} {} {}", new Object[] { packagePath, container, env, cmd }); // TODO: SAMZA-1144 remove the customized approach for package resource and use the common one. // But keep it now for backward compatibility. // set the local package so that the containers and app master are provisioned with it LocalResource packageResource = Records.newRecord(LocalResource.class); URL packageUrl = ConverterUtils.getYarnUrlFromPath(packagePath); FileStatus fileStatus; try { fileStatus = packagePath.getFileSystem(yarnConfiguration).getFileStatus(packagePath); } catch (IOException ioe) { log.error("IO Exception when accessing the package status from the filesystem", ioe); throw new SamzaContainerLaunchException( "IO Exception when accessing the package status from the filesystem"); } packageResource.setResource(packageUrl); log.info("set package Resource in YarnContainerRunner for {}", packageUrl); packageResource.setSize(fileStatus.getLen()); packageResource.setTimestamp(fileStatus.getModificationTime()); packageResource.setType(LocalResourceType.ARCHIVE); packageResource.setVisibility(LocalResourceVisibility.APPLICATION); ByteBuffer allTokens; // copy tokens (copied from dist shell example) try { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // now remove the AM->RM token so that containers cannot access it Iterator iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { TokenIdentifier token = ((Token) iter.next()).decodeIdentifier(); if (token != null && token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } catch (IOException ioe) { log.error("IOException when writing credentials.", ioe); throw new SamzaContainerLaunchException("IO Exception when writing credentials to output buffer"); } Map<String, LocalResource> localResourceMap = new HashMap<>(); localResourceMap.put("__package", packageResource); // include the resources from the universal resource configurations LocalizerResourceMapper resourceMapper = new LocalizerResourceMapper(new LocalizerResourceConfig(config), yarnConfiguration); localResourceMap.putAll(resourceMapper.getResourceMap()); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); context.setEnvironment(env); context.setTokens(allTokens.duplicate()); context.setCommands(new ArrayList<String>() { { add(cmd); } }); context.setLocalResources(localResourceMap); log.debug("setting localResourceMap to {}", localResourceMap); log.debug("setting context to {}", context); StartContainerRequest startContainerRequest = Records.newRecord(StartContainerRequest.class); startContainerRequest.setContainerLaunchContext(context); try { nmClient.startContainer(container, context); } catch (YarnException ye) { log.error("Received YarnException when starting container: " + container.getId(), ye); throw new SamzaContainerLaunchException( "Received YarnException when starting container: " + container.getId(), ye); } catch (IOException ioe) { log.error("Received IOException when starting container: " + container.getId(), ioe); throw new SamzaContainerLaunchException( "Received IOException when starting container: " + container.getId(), ioe); } }
From source file:org.apache.slider.server.appmaster.SliderAppMaster.java
License:Apache License
/** * Async start container request//from w ww . j a va2 s . c om * @param container container * @param ctx context * @param instance node details */ public void startContainer(Container container, ContainerLaunchContext ctx, RoleInstance instance) throws IOException { // Set up tokens for the container too. Today, for normal shell commands, // the container in distribute-shell doesn't need any tokens. We are // populating them mainly for NodeManagers to be able to download any // files in the distributed file-system. The tokens are otherwise also // useful in cases, for e.g., when one is running a "hadoop dfs" command // inside the distributed shell. // add current HDFS delegation token with an up to date token ByteBuffer tokens = getContainerCredentials(); if (tokens != null) { ctx.setTokens(tokens); } else { log.warn("No delegation tokens obtained and set for launch context"); } appState.containerStartSubmitted(container, instance); nmClientAsync.startContainerAsync(container, ctx); }
From source file:org.apache.tajo.master.YarnContainerProxy.java
License:Apache License
public static ContainerLaunchContext createCommonContainerLaunchContext(Configuration config, String queryId, boolean isMaster) { TajoConf conf = (TajoConf) config;// w w w. j av a 2 s . c o m ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class); try { ByteBuffer userToken = ByteBuffer .wrap(UserGroupInformation.getCurrentUser().getShortUserName().getBytes()); ctx.setTokens(userToken); } catch (IOException e) { e.printStackTrace(); } //////////////////////////////////////////////////////////////////////////// // Set the env variables to be setup //////////////////////////////////////////////////////////////////////////// LOG.info("Set the environment for the application master"); Map<String, String> environment = new HashMap<String, String>(); //String initialClassPath = getInitialClasspath(conf); environment.put(ApplicationConstants.Environment.SHELL.name(), "/bin/bash"); if (System.getenv(ApplicationConstants.Environment.JAVA_HOME.name()) != null) { environment.put(ApplicationConstants.Environment.JAVA_HOME.name(), System.getenv(ApplicationConstants.Environment.JAVA_HOME.name())); } // TODO - to be improved with org.apache.tajo.sh shell script Properties prop = System.getProperties(); if (prop.getProperty("tajo.test", "FALSE").equalsIgnoreCase("TRUE") || (System.getenv("tajo.test") != null && System.getenv("tajo.test").equalsIgnoreCase("TRUE"))) { LOG.info("tajo.test is TRUE"); environment.put(ApplicationConstants.Environment.CLASSPATH.name(), prop.getProperty("java.class.path", null)); environment.put("tajo.test", "TRUE"); } else { // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder("./"); //for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH)) { for (String c : YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH) { classPathEnv.append(':'); classPathEnv.append(c.trim()); } classPathEnv.append(":" + System.getenv("TAJO_BASE_CLASSPATH")); classPathEnv.append(":./log4j.properties:./*"); if (System.getenv("HADOOP_HOME") != null) { environment.put("HADOOP_HOME", System.getenv("HADOOP_HOME")); environment.put(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name(), System.getenv("HADOOP_HOME")); environment.put(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name(), System.getenv("HADOOP_HOME")); environment.put(ApplicationConstants.Environment.HADOOP_YARN_HOME.name(), System.getenv("HADOOP_HOME")); } if (System.getenv("TAJO_BASE_CLASSPATH") != null) { environment.put("TAJO_BASE_CLASSPATH", System.getenv("TAJO_BASE_CLASSPATH")); } environment.put(ApplicationConstants.Environment.CLASSPATH.name(), classPathEnv.toString()); } ctx.setEnvironment(environment); if (LOG.isDebugEnabled()) { LOG.debug("================================================="); for (Map.Entry<String, String> entry : environment.entrySet()) { LOG.debug(entry.getKey() + "=" + entry.getValue()); } LOG.debug("================================================="); } //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("defaultFS: " + conf.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)); try { FileSystem fs = FileSystem.get(conf); FileContext fsCtx = FileContext.getFileContext(conf); Path systemConfPath = TajoConf.getSystemConfPath(conf); if (!fs.exists(systemConfPath)) { LOG.error("system_conf.xml (" + systemConfPath.toString() + ") Not Found"); } LocalResource systemConfResource = createApplicationResource(fsCtx, systemConfPath, LocalResourceType.FILE); localResources.put(TajoConstants.SYSTEM_CONF_FILENAME, systemConfResource); ctx.setLocalResources(localResources); } catch (IOException e) { LOG.error(e.getMessage(), e); } Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>(); try { serviceData.put(PullServerAuxService.PULLSERVER_SERVICEID, PullServerAuxService.serializeMetaData(0)); } catch (IOException ioe) { LOG.error(ioe); } ctx.setServiceData(serviceData); return ctx; }
From source file:org.apache.tajo.yarn.command.LaunchCommand.java
License:Apache License
private void setupSecurityTokens(ContainerLaunchContext amContainer, FileSystem fs) throws IOException { if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); }/*from w w w .ja v a 2s .co m*/ // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } }
From source file:org.apache.tajo.yarn.container.WorkerContainerTask.java
License:Apache License
@Override public ContainerLaunchContext getLaunchContext(Container container) throws IOException { // create a container launch context ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class); UserGroupInformation user = UserGroupInformation.getCurrentUser(); try {/*from ww w. ja v a2 s. c o m*/ Credentials credentials = user.getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); launchContext.setTokens(securityTokens); } catch (IOException e) { LOG.warn("Getting current user info failed when trying to launch the container" + e.getMessage()); } FileSystem fs = FileSystem.get(appContext.getConfiguration()); // Set the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); String suffix = "Tajo" + "/" + appContext.getApplicationId().getId(); Path parentPath = new Path(fs.getHomeDirectory(), suffix); // tar ball Path archivePath = new Path(parentPath, System.getenv(Constants.TAJO_ARCHIVE_PATH)); FileStatus archiveFs = fs.getFileStatus(archivePath); LocalResource archiveRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(archivePath.toUri()), LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION, archiveFs.getLen(), archiveFs.getModificationTime()); localResources.put("tajo", archiveRsrc); Configuration tajoWorkerConf = new Configuration(false); tajoWorkerConf.addResource(new Path("conf", "tajo-site.xml")); tajoWorkerConf.set(Constants.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, appContext.getMasterHost() + ":26001"); tajoWorkerConf.set(Constants.CATALOG_ADDRESS, appContext.getMasterHost() + ":26005"); Path dst = new Path(parentPath, container.getId() + Path.SEPARATOR + "worker-conf"); fs.mkdirs(dst); Path confFile = new Path(dst, "tajo-site.xml"); FSDataOutputStream fdos = fs.create(confFile); tajoWorkerConf.writeXml(fdos); fdos.close(); FileStatus scFileStatus = fs.getFileStatus(dst); LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime()); localResources.put("conf", scRsrc); launchContext.setLocalResources(localResources); // Set the environment setupEnv(launchContext); // Set the necessary command to execute on the allocated container Vector<CharSequence> vargs = new Vector<CharSequence>(5); // Set executable command // Set args for the shell command if any vargs.add("${" + Constants.TAJO_HOME + "}/bin/tajo"); vargs.add("--config"); vargs.add("${" + Constants.TAJO_CONF_DIR + "}"); vargs.add("worker"); // Add log redirect params // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } List<String> commands = new ArrayList<String>(); commands.add(command.toString()); launchContext.setCommands(commands); return launchContext; }
From source file:org.apache.twill.internal.yarn.Hadoop23YarnAppClient.java
License:Apache License
/** * Overrides parent method to adds RM delegation token to the given context. If YARN is running with HA RM, * delegation tokens for each RM service will be added. */// w w w . j a v a 2s . com protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) { if (!UserGroupInformation.isSecurityEnabled()) { return; } try { Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName()); org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient.getRMDelegationToken(renewer); // The following logic is copied from ClientRMProxy.getRMDelegationTokenService, which is not available in // YARN older than 2.4 List<String> services = new ArrayList<>(); if (HAUtil.isHAEnabled(configuration)) { // If HA is enabled, we need to enumerate all RM hosts // and add the corresponding service name to the token service // Copy the yarn conf since we need to modify it to get the RM addresses YarnConfiguration yarnConf = new YarnConfiguration(configuration); for (String rmId : HAUtil.getRMHAIds(configuration)) { yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); services.add(SecurityUtil.buildTokenService(address).toString()); } } else { services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString()); } Credentials credentials = YarnUtils.decodeCredentials(context.getTokens()); // casting needed for later Hadoop version @SuppressWarnings("RedundantCast") Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken, (InetSocketAddress) null); token.setService(new Text(Joiner.on(',').join(services))); credentials.addToken(new Text(token.getService()), token); LOG.debug("Added RM delegation token {} for application {}", token, appId); credentials.addToken(token.getService(), token); context.setTokens(YarnUtils.encodeCredentials(credentials)); } catch (Exception e) { throw Throwables.propagate(e); } }