List of usage examples for org.apache.hadoop.yarn.api ApplicationConstants STDERR
String STDERR
To view the source code for org.apache.hadoop.yarn.api ApplicationConstants STDERR.
Click Source Link
From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process) * @param container the samza container to run. * @param cmdBuilder the command builder that encapsulates the command, and the context * * @throws SamzaContainerLaunchException when there's an exception in submitting the request to the RM. * *//*from w w w . ja v a2s. c o m*/ //TODO: we don't need samzaContainerId as a param here. public void runContainer(int samzaContainerId, Container container, CommandBuilder cmdBuilder) throws SamzaContainerLaunchException { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib); cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); printContainerEnvironmentVariables(samzaContainerId, env); log.info("Samza FWK path: " + command + "; env=" + env); Path path = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, path); startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param processorId id of the samza processor to run (passed as a command line parameter to the process) * @param container the yarn container to run the processor on. * @param cmdBuilder the command builder that encapsulates the command, and the context * @throws IOException on IO exceptions running the container *///from w ww . ja v a 2 s.c o m public void runProcessor(String processorId, Container container, CommandBuilder cmdBuilder) throws IOException { String containerIdStr = ConverterUtils.toString(container.getId()); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(this.config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString())); Path packagePath = new Path(yarnConfig.getPackagePath()); String formattedCommand = getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR); log.info( "Running Processor ID: {} on Container ID: {} on host: {} using command: {} and env: {} and package path: {}", processorId, containerIdStr, container.getNodeHttpAddress(), formattedCommand, env, packagePath); state.pendingProcessors.put(processorId, new YarnContainer(container)); startContainer(packagePath, container, env, formattedCommand); log.info( "Made start request for Processor ID: {} on Container ID: {} on host: {} (http://{}/node/containerlogs/{}).", processorId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr); }
From source file:org.apache.samza.job.yarn.YarnContainerRunner.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process) * @param container the samza container to run. * @param cmdBuilder the command builder that encapsulates the command, and the context * * @throws SamzaContainerLaunchException when there's an exception in submitting the request to the RM. * *///from www. j a va 2s . c om public void runContainer(String samzaContainerId, Container container, CommandBuilder cmdBuilder) throws SamzaContainerLaunchException { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib); cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString())); printContainerEnvironmentVariables(samzaContainerId, env); log.info("Samza FWK path: " + command + "; env=" + env); Path packagePath = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, packagePath); startContainer(packagePath, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }
From source file:org.apache.tez.client.TezClientUtils.java
License:Apache License
/** * Create an ApplicationSubmissionContext to launch a Tez AM * @param appId Application Id/*from w ww . j a va2s . c om*/ * @param dag DAG to be submitted * @param amName Name for the application * @param amConfig AM Configuration * @param tezJarResources Resources to be used by the AM * @param sessionCreds the credential object which will be populated with session specific * @param historyACLPolicyManager * @return an ApplicationSubmissionContext to launch a Tez AM * @throws IOException * @throws YarnException */ @Private @VisibleForTesting public static ApplicationSubmissionContext createApplicationSubmissionContext(ApplicationId appId, DAG dag, String amName, AMConfiguration amConfig, Map<String, LocalResource> tezJarResources, Credentials sessionCreds, boolean tezLrsAsArchive, TezApiVersionInfo apiVersionInfo, HistoryACLPolicyManager historyACLPolicyManager) throws IOException, YarnException { Preconditions.checkNotNull(sessionCreds); TezConfiguration conf = amConfig.getTezConfiguration(); FileSystem fs = TezClientUtils.ensureStagingDirExists(conf, TezCommonUtils.getTezBaseStagingPath(conf)); String strAppId = appId.toString(); Path tezSysStagingPath = TezCommonUtils.createTezSystemStagingPath(conf, strAppId); Path binaryConfPath = TezCommonUtils.getTezConfStagingPath(tezSysStagingPath); binaryConfPath = fs.makeQualified(binaryConfPath); // Setup resource requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT)); capability.setVirtualCores(amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT)); if (LOG.isDebugEnabled()) { LOG.debug("AppMaster capability = " + capability); } // Setup required Credentials for the AM launch. DAG specific credentials // are handled separately. ByteBuffer securityTokens = null; // Setup security tokens Credentials amLaunchCredentials = new Credentials(); if (amConfig.getCredentials() != null) { amLaunchCredentials.addAll(amConfig.getCredentials()); } // Add Staging dir creds to the list of session credentials. TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { binaryConfPath }, conf); // Add session specific credentials to the AM credentials. amLaunchCredentials.mergeAll(sessionCreds); DataOutputBuffer dob = new DataOutputBuffer(); amLaunchCredentials.writeTokenStorageToStream(dob); securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Setup the command to run the AM List<String> vargs = new ArrayList<String>(8); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); String amOpts = constructAMLaunchOpts(amConfig.getTezConfiguration(), capability); vargs.add(amOpts); String amLogLevelString = amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_LOG_LEVEL, TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT); String[] amLogParams = parseLogParams(amLogLevelString); String amLogLevel = amLogParams[0]; maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs); // FIX sun bug mentioned in TEZ-327 vargs.add("-Dsun.nio.ch.bugLevel=''"); vargs.add(TezConstants.TEZ_APPLICATION_MASTER_CLASS); if (dag == null) { vargs.add("--" + TezConstants.TEZ_SESSION_MODE_CLI_OPTION); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDOUT); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR); Vector<String> vargsFinal = new Vector<String>(8); // Final command StringBuilder mergedCommand = new StringBuilder(); for (CharSequence str : vargs) { mergedCommand.append(str).append(" "); } vargsFinal.add(mergedCommand.toString()); if (LOG.isDebugEnabled()) { LOG.debug("Command to launch container for ApplicationMaster is : " + mergedCommand); } Map<String, String> environment = new TreeMap<String, String>(); TezYARNUtils.setupDefaultEnv(environment, conf, TezConfiguration.TEZ_AM_LAUNCH_ENV, TezConfiguration.TEZ_AM_LAUNCH_ENV_DEFAULT, tezLrsAsArchive); addVersionInfoToEnv(environment, apiVersionInfo); addLogParamsToEnv(environment, amLogParams); Map<String, LocalResource> amLocalResources = new TreeMap<String, LocalResource>(); // Not fetching credentials for AMLocalResources. Expect this to be provided via AMCredentials. if (amConfig.getAMLocalResources() != null) { amLocalResources.putAll(amConfig.getAMLocalResources()); } amLocalResources.putAll(tezJarResources); // Setup Session ACLs and update conf as needed Map<String, String> aclConfigs = null; if (historyACLPolicyManager != null) { if (dag == null) { aclConfigs = historyACLPolicyManager.setupSessionACLs(amConfig.getTezConfiguration(), appId); } else { // Non-session mode // As only a single DAG is support, we should combine AM and DAG ACLs under the same // acl management layer aclConfigs = historyACLPolicyManager.setupNonSessionACLs(amConfig.getTezConfiguration(), appId, dag.getDagAccessControls()); } } // emit conf as PB file ConfigurationProto finalConfProto = createFinalConfProtoForApp(amConfig.getTezConfiguration(), aclConfigs); FSDataOutputStream amConfPBOutBinaryStream = null; try { amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath); finalConfProto.writeTo(amConfPBOutBinaryStream); } finally { if (amConfPBOutBinaryStream != null) { amConfPBOutBinaryStream.close(); } } LocalResource binaryConfLRsrc = TezClientUtils.createLocalResource(fs, binaryConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); amConfig.setBinaryConfLR(binaryConfLRsrc); amLocalResources.put(TezConstants.TEZ_PB_BINARY_CONF_NAME, binaryConfLRsrc); // Create Session Jars definition to be sent to AM as a local resource Path sessionJarsPath = TezCommonUtils.getTezAMJarStagingPath(tezSysStagingPath); FSDataOutputStream sessionJarsPBOutStream = null; try { sessionJarsPBOutStream = TezCommonUtils.createFileForAM(fs, sessionJarsPath); // Write out the initial list of resources which will be available in the AM DAGProtos.PlanLocalResourcesProto amResourceProto; if (amLocalResources != null && !amLocalResources.isEmpty()) { amResourceProto = DagTypeConverters.convertFromLocalResources(amLocalResources); } else { amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance(); } amResourceProto.writeDelimitedTo(sessionJarsPBOutStream); } finally { if (sessionJarsPBOutStream != null) { sessionJarsPBOutStream.close(); } } LocalResource sessionJarsPBLRsrc = TezClientUtils.createLocalResource(fs, sessionJarsPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); amLocalResources.put(TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, sessionJarsPBLRsrc); String user = UserGroupInformation.getCurrentUser().getShortUserName(); ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration()); Map<ApplicationAccessType, String> acls = aclManager.toYARNACls(); if (dag != null) { DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, sessionCreds); // emit protobuf DAG file style Path binaryPath = TezCommonUtils.getTezBinPlanStagingPath(tezSysStagingPath); if (LOG.isDebugEnabled()) { LOG.debug("Stage directory information for AppId :" + appId + " tezSysStagingPath :" + tezSysStagingPath + " binaryConfPath :" + binaryConfPath + " sessionJarsPath :" + sessionJarsPath + " binaryPlanPath :" + binaryPath); } FSDataOutputStream dagPBOutBinaryStream = null; try { //binary output dagPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryPath); dagPB.writeTo(dagPBOutBinaryStream); } finally { if (dagPBOutBinaryStream != null) { dagPBOutBinaryStream.close(); } } amLocalResources.put(TezConstants.TEZ_PB_PLAN_BINARY_NAME, TezClientUtils.createLocalResource(fs, binaryPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) { Path textPath = localizeDagPlanAsText(dagPB, fs, amConfig, strAppId, tezSysStagingPath); amLocalResources.put(TezConstants.TEZ_PB_PLAN_TEXT_NAME, TezClientUtils.createLocalResource(fs, textPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); } } // Setup ContainerLaunchContext for AM container ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(amLocalResources, environment, vargsFinal, null, securityTokens, acls); // Set up the ApplicationSubmissionContext ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE); appContext.setApplicationId(appId); appContext.setResource(capability); if (amConfig.getQueueName() != null) { appContext.setQueue(amConfig.getQueueName()); } appContext.setApplicationName(amName); appContext.setCancelTokensWhenComplete(amConfig.getTezConfiguration().getBoolean( TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION, TezConfiguration.TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT)); appContext.setAMContainerSpec(amContainer); appContext.setMaxAppAttempts(amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT)); return appContext; }
From source file:org.deeplearning4j.iterativereduce.runtime.Utils.java
License:Apache License
private static StringBuffer getCommandsBase(Configuration conf, Properties props, String command, String args) { StringBuffer sb = new StringBuffer(); sb.append("java "); sb.append("-Xmx").append(props.getProperty(ConfigFields.YARN_MEMORY, "512")).append("m "); if (args != null) sb.append(" ").append(args).append(" "); // Actual command sb.append(command);//from w ww .j a v a 2 s .c o m sb.append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(Path.SEPARATOR) .append(ApplicationConstants.STDOUT); sb.append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(Path.SEPARATOR) .append(ApplicationConstants.STDERR); return sb; }
From source file:org.elasticsearch.hadoop.yarn.am.EsCluster.java
License:Apache License
private List<String> setupEsScript(Config conf) { List<String> cmds = new ArrayList<String>(); // don't use -jar since it overrides the classpath cmds.add(YarnCompat.$$(ApplicationConstants.Environment.SHELL)); // make sure to include the ES.ZIP archive name used in the local resource setup above (since it's the folder where it got unpacked) cmds.add(conf.esZipName() + "/" + conf.esScript()); cmds.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + ApplicationConstants.STDOUT); cmds.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + ApplicationConstants.STDERR); return Collections.singletonList(StringUtils.concatenate(cmds, " ")); }
From source file:org.elasticsearch.hadoop.yarn.client.YarnLauncher.java
License:Apache License
private List<String> setupCmd() { List<String> cmds = new ArrayList<String>(); // don't use -jar since it overrides the classpath cmds.add(YarnCompat.$$(ApplicationConstants.Environment.JAVA_HOME) + "/bin/java"); cmds.add(ApplicationMaster.class.getName()); cmds.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + ApplicationConstants.STDOUT); cmds.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + ApplicationConstants.STDERR); return Collections.singletonList(StringUtils.concatenate(cmds, " ")); }
From source file:org.hdl.tensorflow.yarn.appmaster.LaunchContainerThread.java
License:Apache License
private String makeContainerCommand(long containerMemory, String clusterSpec, String jobName, int taskIndex) { String[] commands = new String[] { ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java", "-Xmx" + containerMemory + "m", TFServerRunner.class.getName() + " ", Utils.mkOption(Constants.OPT_CLUSTER_SPEC, clusterSpec), Utils.mkOption(Constants.OPT_JOB_NAME, jobName), Utils.mkOption(Constants.OPT_TASK_INDEX, taskIndex), "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." + ApplicationConstants.STDOUT, "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." + ApplicationConstants.STDERR }; return Utils.mkString(commands, " "); }