List of usage examples for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR
String LOG_DIR_EXPANSION_VAR
To view the source code for org.apache.hadoop.yarn.api ApplicationConstants LOG_DIR_EXPANSION_VAR.
Click Source Link
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
/** * Main run function for the client/*w ww.j a v a 2s. c o m*/ * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } if (domainId != null && domainId.length() > 0 && toCreateDomain) { prepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts if (domainId != null && domainId.length() > 0) { env.put(Constants.TIMELINEDOMAIN, domainId); } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add(ApplicationMaster.AMOptions.toArgs(ApplicationMaster.AMOptions.ZK_QUORUM.of(zkQuorum), ApplicationMaster.AMOptions.ZK_ROOT.of(zkRoot), ApplicationMaster.AMOptions.APP_JAR_PATH.of(ajPath.toString()))); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application return monitorApplication(appId); }
From source file:org.apache.reef.runtime.hdinsight.client.HDInsightJobSubmissionHandler.java
License:Apache License
/** * Assembles the command to execute the Driver in list form. */// w ww. j a v a2 s.c o m private List<String> getCommandList(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { return new JavaLaunchCommandBuilder().setJavaPath("%JAVA_HOME%/bin/java") .setErrorHandlerRID(jobSubmissionProto.getRemoteId()) .setLaunchID(jobSubmissionProto.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath()).setMemory(jobSubmissionProto.getDriverMemory()) .setStandardErr( ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName()) .setStandardOut( ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName()) .build(); }
From source file:org.apache.reef.runtime.yarn.client.YarnJobSubmissionHandler.java
License:Apache License
@Override public void onNext(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { LOG.log(Level.FINEST, "Submitting job with ID [{0}]", jobSubmissionProto.getIdentifier()); try {// w w w . j a va 2s.co m LOG.log(Level.FINE, "Requesting Application ID from YARN."); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); final GetNewApplicationResponse applicationResponse = yarnClientApplication.getNewApplicationResponse(); final ApplicationSubmissionContext applicationSubmissionContext = yarnClientApplication .getApplicationSubmissionContext(); final ApplicationId applicationId = applicationSubmissionContext.getApplicationId(); LOG.log(Level.FINEST, "YARN Application ID: {0}", applicationId); // set the application name applicationSubmissionContext.setApplicationName("reef-job-" + jobSubmissionProto.getIdentifier()); LOG.log(Level.FINE, "Assembling submission JAR for the Driver."); final Path submissionFolder = new Path( "/tmp/" + this.filenames.getJobFolderPrefix() + applicationId.getId() + "/"); final Configuration driverConfiguration = makeDriverConfiguration(jobSubmissionProto, submissionFolder); final File jobSubmissionFile = this.jobJarMaker.createJobSubmissionJAR(jobSubmissionProto, driverConfiguration); final Path uploadedJobJarPath = this.uploadToJobFolder(jobSubmissionFile, submissionFolder); final Map<String, LocalResource> resources = new HashMap<>(1); resources.put(this.filenames.getREEFFolderName(), this.makeLocalResourceForJarFile(uploadedJobJarPath)); // SET MEMORY RESOURCE final int amMemory = getMemory(jobSubmissionProto, applicationResponse.getMaximumResourceCapability().getMemory()); applicationSubmissionContext.setResource(Resource.newInstance(amMemory, 1)); // SET EXEC COMMAND final List<String> launchCommand = new JavaLaunchCommandBuilder() .setErrorHandlerRID(jobSubmissionProto.getRemoteId()) .setLaunchID(jobSubmissionProto.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath()).setMemory(amMemory) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName()) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName()) .build(); applicationSubmissionContext .setAMContainerSpec(YarnTypes.getContainerLaunchContext(launchCommand, resources)); applicationSubmissionContext.setPriority(getPriority(jobSubmissionProto)); // Set the queue to which this application is to be submitted in the RM applicationSubmissionContext.setQueue(getQueue(jobSubmissionProto, "default")); LOG.log(Level.INFO, "Submitting REEF Application to YARN. ID: {0}", applicationId); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "REEF app command: {0}", StringUtils.join(launchCommand, ' ')); } // TODO: this is currently being developed on a hacked 2.4.0 bits, should be 2.4.1 final String minVersionKeepContainerOptionAvailable = "2.4.0"; // when supported, set KeepContainersAcrossApplicationAttempts to be true // so that when driver (AM) crashes, evaluators will still be running and we can recover later. if (YarnTypes.isAtOrAfterVersion(minVersionKeepContainerOptionAvailable)) { LOG.log(Level.FINE, "Hadoop version is {0} or after with KeepContainersAcrossApplicationAttempts supported, will set it to true.", minVersionKeepContainerOptionAvailable); applicationSubmissionContext.setKeepContainersAcrossApplicationAttempts(true); } this.yarnClient.submitApplication(applicationSubmissionContext); } catch (final YarnException | IOException e) { throw new RuntimeException("Unable to submit Driver to YARN.", e); } }
From source file:org.apache.reef.runtime.yarn.client.YarnSubmissionHelper.java
License:Apache License
public void submit() throws IOException, YarnException { // SET EXEC COMMAND final List<String> launchCommand = new JavaLaunchCommandBuilder(launcherClazz, commandPrefixList) .setConfigurationFileName(confFileName).setClassPath(this.classpath.getDriverClasspath()) .setMemory(this.applicationSubmissionContext.getResource().getMemory()) .setStandardOut(//w w w . j a v a 2 s . c o m ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.fileNames.getDriverStdoutFileName()) .setStandardErr( ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.fileNames.getDriverStderrFileName()) .build(); if (this.applicationSubmissionContext.getKeepContainersAcrossApplicationAttempts() && this.applicationSubmissionContext.getMaxAppAttempts() == 1) { LOG.log(Level.WARNING, "Application will not be restarted even though preserve evaluators is set to true" + " since the max application submissions is 1. Proceeding to submit application..."); } final ContainerLaunchContext containerLaunchContext = YarnTypes.getContainerLaunchContext(launchCommand, this.resources, tokenProvider.getTokens()); this.applicationSubmissionContext.setAMContainerSpec(containerLaunchContext); LOG.log(Level.INFO, "Submitting REEF Application to YARN. ID: {0}", this.applicationId); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "REEF app command: {0}", StringUtils.join(launchCommand, ' ')); } this.yarnClient.submitApplication(applicationSubmissionContext); }
From source file:org.apache.reef.runtime.yarn.driver.YARNResourceLaunchHandler.java
License:Apache License
@Override public void onNext(final DriverRuntimeProtocol.ResourceLaunchProto resourceLaunchProto) { try {//w ww .jav a2 s. c om final String containerId = resourceLaunchProto.getIdentifier(); LOG.log(Level.FINEST, "TIME: Start ResourceLaunchProto {0}", containerId); final Container container = this.containers.get(containerId); LOG.log(Level.FINEST, "Setting up container launch container for id={0}", container.getId()); final Map<String, LocalResource> localResources = this.evaluatorSetupHelper .getResources(resourceLaunchProto); final LaunchCommandBuilder commandBuilder; switch (resourceLaunchProto.getType()) { case JVM: commandBuilder = new JavaLaunchCommandBuilder() .setClassPath(this.classpath.getEvaluatorClasspath()); break; case CLR: commandBuilder = new CLRLaunchCommandBuilder(); break; default: throw new IllegalArgumentException("Unsupported container type: " + resourceLaunchProto.getType()); } final List<String> command = commandBuilder.setErrorHandlerRID(resourceLaunchProto.getRemoteId()) .setLaunchID(resourceLaunchProto.getIdentifier()) .setConfigurationFileName(this.filenames.getEvaluatorConfigurationPath()) .setMemory((int) (this.jvmHeapFactor * container.getResource().getMemory())) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getEvaluatorStderrFileName()) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getEvaluatorStdoutFileName()) .build(); if (LOG.isLoggable(Level.FINEST)) { LOG.log(Level.FINEST, "TIME: Run ResourceLaunchProto {0} command: `{1}` with resources: `{2}`", new Object[] { containerId, StringUtils.join(command, ' '), localResources }); } final ContainerLaunchContext ctx = YarnTypes.getContainerLaunchContext(command, localResources); this.yarnContainerManager.get().submit(container, ctx); LOG.log(Level.FINEST, "TIME: End ResourceLaunchProto {0}", containerId); } catch (final Throwable e) { LOG.log(Level.WARNING, "Error handling resource launch message: " + resourceLaunchProto, e); throw new RuntimeException(e); } }
From source file:org.apache.samza.job.yarn.ContainerUtil.java
License:Apache License
public void runContainer(int samzaContainerId, Container container) { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); String cmdBuilderClassName;/*from w ww . ja v a2 s . c o m*/ if (taskConfig.getCommandClass().isDefined()) { cmdBuilderClassName = taskConfig.getCommandClass().get(); } else { cmdBuilderClassName = ShellCommandBuilder.class.getName(); } CommandBuilder cmdBuilder = (CommandBuilder) Util.getObj(cmdBuilderClassName); cmdBuilder.setConfig(config).setId(samzaContainerId).setUrl(state.coordinatorUrl); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); log.info("Container ID {} using environment variables: ", samzaContainerId); Map<String, String> env = new HashMap<String, String>(); for (Map.Entry<String, String> entry : cmdBuilder.buildEnvironment().entrySet()) { String escapedValue = Util.envVarEscape(entry.getValue()); env.put(entry.getKey(), escapedValue); log.info("{}={} ", entry.getKey(), escapedValue); } Path path = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, path); startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); if (state.neededContainers.decrementAndGet() == 0) { state.jobHealthy.set(true); } state.runningContainers.put(samzaContainerId, new YarnContainer(container)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }
From source file:org.apache.samza.job.yarn.refactor.YarnContainerRunner.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process) * @param container the samza container to run. * @param cmdBuilder the command builder that encapsulates the command, and the context * * @throws SamzaContainerLaunchException when there's an exception in submitting the request to the RM. * *///from w w w.j av a2 s.c om //TODO: we don't need samzaContainerId as a param here. public void runContainer(int samzaContainerId, Container container, CommandBuilder cmdBuilder) throws SamzaContainerLaunchException { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib); cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); printContainerEnvironmentVariables(samzaContainerId, env); log.info("Samza FWK path: " + command + "; env=" + env); Path path = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, path); startContainer(path, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }
From source file:org.apache.samza.job.yarn.YarnClusterResourceManager.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param processorId id of the samza processor to run (passed as a command line parameter to the process) * @param container the yarn container to run the processor on. * @param cmdBuilder the command builder that encapsulates the command, and the context * @throws IOException on IO exceptions running the container *//*from ww w .j av a2 s . co m*/ public void runProcessor(String processorId, Container container, CommandBuilder cmdBuilder) throws IOException { String containerIdStr = ConverterUtils.toString(container.getId()); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(this.config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString())); Path packagePath = new Path(yarnConfig.getPackagePath()); String formattedCommand = getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR); log.info( "Running Processor ID: {} on Container ID: {} on host: {} using command: {} and env: {} and package path: {}", processorId, containerIdStr, container.getNodeHttpAddress(), formattedCommand, env, packagePath); state.pendingProcessors.put(processorId, new YarnContainer(container)); startContainer(packagePath, container, env, formattedCommand); log.info( "Made start request for Processor ID: {} on Container ID: {} on host: {} (http://{}/node/containerlogs/{}).", processorId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr); }
From source file:org.apache.samza.job.yarn.YarnContainerRunner.java
License:Apache License
/** * Runs a process as specified by the command builder on the container. * @param samzaContainerId id of the samza Container to run (passed as a command line parameter to the process) * @param container the samza container to run. * @param cmdBuilder the command builder that encapsulates the command, and the context * * @throws SamzaContainerLaunchException when there's an exception in submitting the request to the RM. * *///from w w w.j a v a 2 s .c o m public void runContainer(String samzaContainerId, Container container, CommandBuilder cmdBuilder) throws SamzaContainerLaunchException { String containerIdStr = ConverterUtils.toString(container.getId()); log.info("Got available container ID ({}) for container: {}", samzaContainerId, container); // check if we have framework path specified. If yes - use it, if not use default ./__package/ String jobLib = ""; // in case of separate framework, this directory will point at the job's libraries String cmdPath = "./__package/"; String fwkPath = JobConfig.getFwkPath(config); if (fwkPath != null && (!fwkPath.isEmpty())) { cmdPath = fwkPath; jobLib = "export JOB_LIB_DIR=./__package/lib"; } log.info("In runContainer in util: fwkPath= " + fwkPath + ";cmdPath=" + cmdPath + ";jobLib=" + jobLib); cmdBuilder.setCommandPath(cmdPath); String command = cmdBuilder.buildCommand(); log.info("Container ID {} using command {}", samzaContainerId, command); Map<String, String> env = getEscapedEnvironmentVariablesMap(cmdBuilder); env.put(ShellCommandConfig.ENV_EXECUTION_ENV_CONTAINER_ID(), Util.envVarEscape(container.getId().toString())); printContainerEnvironmentVariables(samzaContainerId, env); log.info("Samza FWK path: " + command + "; env=" + env); Path packagePath = new Path(yarnConfig.getPackagePath()); log.info("Starting container ID {} using package path {}", samzaContainerId, packagePath); startContainer(packagePath, container, env, getFormattedCommand(ApplicationConstants.LOG_DIR_EXPANSION_VAR, jobLib, command, ApplicationConstants.STDOUT, ApplicationConstants.STDERR)); log.info("Claimed container ID {} for container {} on node {} (http://{}/node/containerlogs/{}).", new Object[] { samzaContainerId, containerIdStr, container.getNodeId().getHost(), container.getNodeHttpAddress(), containerIdStr }); log.info("Started container ID {}", samzaContainerId); }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * * @param clustername name of the cluster * @param clusterDirectory cluster dir/*from www . j av a 2 s . c o m*/ * @param instanceDefinition the instance definition * @param debugAM enable debug AM options * @return the launched application * @throws YarnException * @throws IOException */ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { deployedClusterName = clustername; SliderUtils.validateClusterName(clustername); verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config); //create the Slider AM provider -this helps set up the AM SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME)); // make sure the conf dir is valid; if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, appOperations.getGlobalOptions().get(AgentKeys.APP_DEF)); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config, sliderFileSystem, yarnClient, clusterSecure, sliderAMResourceComponent, resourceGlobalOptions, applicationTags); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); amLauncher.setMaxAppAttempts(maxAppAttempts); sliderFileSystem.purgeAppInstanceTempFiles(clustername); Path tempPath = sliderFileSystem.createAppInstanceTempPath(clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); sliderFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(), tempPath, libPath); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); // look for the configuration directory named on the command line boolean hasServerLog4jProperties = false; Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); if (confdirProp == null || confdirProp.isEmpty()) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); if (!confDir.exists()) { throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } Path localConfDirPath = SliderUtils.createLocalPath(confDir); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath); SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); hasServerLog4jProperties = log4jserver.isFile(); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself boolean usingMiniMRCluster = getUsingMiniMRCluster(); if (!usingMiniMRCluster) { log.debug("Destination is not a MiniYARNCluster -copying full classpath"); // insert conf dir first if (remoteConfPath != null) { relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); SliderUtils.mergeMaps(localResources, submittedConfDir); } } // build up the configuration // IMPORTANT: it is only after this call that site configurations // will be valid. propagatePrincipals(config, instanceDefinition); // validate security data /* // turned off until tested SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, instanceDefinition, clustername); */ Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); // standard AM resources sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); //add provider-specific resources provider.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); // now that the site config is fully generated, the provider gets // to do a quick review of them. log.debug("Preflight validation of cluster configuration"); sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); provider.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); // TODO: consider supporting apps that don't have an image path Path imagePath = SliderUtils.extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv(SliderUtils.buildEnvMap(sliderAMResourceComponent)); ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(), usingMiniMRCluster); amLauncher.setClasspath(classpath); if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", SliderUtils.stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); } // rm address InetSocketAddress rmSchedulerAddress; try { rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS, config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder(); // insert any JVM options); sliderAM.addJVMOptions(instanceDefinition, commandLine); // enable asserts if the text option is set commandLine.enableJavaAssertions(); // if the conf dir has a log4j-server.properties, switch to that if (hasServerLog4jProperties) { commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); } // add the AM sevice entry point commandLine.add(SliderAppMaster.SERVICE_CLASSNAME); // create action and the cluster name commandLine.add(ACTION_CREATE, clustername); // debug if (debugAM) { commandLine.add(Arguments.ARG_DEBUG); } // set the cluster directory path commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri()); if (!isUnset(rmAddr)) { commandLine.add(Arguments.ARG_RM_ADDR, rmAddr); } if (serviceArgs.getFilesystemBinding() != null) { commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); } /** * pass the registry binding */ addConfOptionToCLI(commandLine, config, REGISTRY_PATH, DEFAULT_REGISTRY_PATH); addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over /* addConfOptionToCLI(commandLine, config, KEY_SECURITY); */ addConfOptionToCLI(commandLine, config, DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = commandLine.build(); log.debug("Completed setting up app master command {}", cmdStr); amLauncher.addCommandLine(commandLine); // the Slider AM gets to configure the AM requirements, not the custom provider sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, amLauncher.getResource()); // Set the priority for the application master int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY); amLauncher.setPriority(amPriority); // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE); if (!SliderUtils.isUnset(suppliedQueue)) { amQueue = suppliedQueue; log.info("Using queue {} for the application instance.", amQueue); } if (amQueue != null) { amLauncher.setQueue(amQueue); } // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; }