List of usage examples for org.apache.hadoop.yarn.api.records ApplicationId toString
@Override
public String toString()
From source file:org.apache.metron.maas.service.Client.java
License:Apache License
/** * Main run function for the client/*from ww w .ja va 2s.c o m*/ * @return true if application completed successfully * @throws IOException * @throws YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING); LOG.info("Got Cluster node info from ASM"); for (NodeReport node : clusterNodeReports) { LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress" + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers" + node.getNumContainers()); } QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() + ", queueApplicationCount=" + queueInfo.getApplications().size() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl=" + userAcl.name()); } } if (domainId != null && domainId.length() > 0 && toCreateDomain) { prepareTimelineDomain(); } // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); appContext.setApplicationName(appName); if (attemptFailuresValidityInterval >= 0) { appContext.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval); } // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); LOG.info("Copy App Master jar from local filesystem and add to local environment"); // Copy the application master jar to the filesystem // Create a local resource to point to the destination jar path FileSystem fs = FileSystem.get(conf); Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), localResources, null); } // Set the necessary security tokens as needed //amContainer.setContainerTokens(containerToken); // Set the env variables to be setup in the env where the application master will be run LOG.info("Set the environment for the application master"); Map<String, String> env = new HashMap<String, String>(); // put location of shell script into env // using the env info, the application master will create the correct local resource for the // eventual containers that will be launched to execute the shell scripts if (domainId != null && domainId.length() > 0) { env.put(Constants.TIMELINEDOMAIN, domainId); } // Add AppMaster.jar location to classpath // At some point we should not be required to add // the hadoop specific classpaths to the env. // It should be provided out of the box. // For now setting all required classpaths including // the classpath to "." for the application jar StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); classPathEnv.append(c.trim()); } classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./log4j.properties"); // add the runtime classpath needed for tests to work if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); } env.put("CLASSPATH", classPathEnv.toString()); // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command LOG.info("Setting up app master command"); vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx" + amMemory + "m"); // Set class name vargs.add(appMasterMainClass); // Set params for Application Master vargs.add(ApplicationMaster.AMOptions.toArgs(ApplicationMaster.AMOptions.ZK_QUORUM.of(zkQuorum), ApplicationMaster.AMOptions.ZK_ROOT.of(zkRoot), ApplicationMaster.AMOptions.APP_JAR_PATH.of(ajPath.toString()))); if (null != nodeLabelExpression) { appContext.setNodeLabelExpression(nodeLabelExpression); } for (Map.Entry<String, String> entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up app master command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); // Set up the container launch context for the application master ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Resource.newInstance(amMemory, amVCores); appContext.setResource(capability); // Service data is a binary blob that can be passed to the application // Not needed in this scenario // Setup security tokens if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOG.info("Got dt for " + fs.getUri() + "; " + token); } } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens); } appContext.setAMContainerSpec(amContainer); // Set the priority for the application master Priority pri = Priority.newInstance(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application return monitorApplication(appId); }
From source file:org.apache.oozie.action.hadoop.LauncherAM.java
License:Apache License
private void setRecoveryId() throws LauncherException { try {/*from w ww .j a va2 s .co m*/ ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId(); final String applicationIdStr = applicationId.toString(); final String recoveryId = Objects.requireNonNull(launcherConf.get(OOZIE_ACTION_RECOVERY_ID), "RecoveryID should not be null"); final Path path = new Path(actionDir, recoveryId); if (!hdfsOperations.fileExists(path, launcherConf)) { hdfsOperations.writeStringToFile(path, launcherConf, applicationIdStr); } else { final String id = hdfsOperations.readFileContents(path, launcherConf); if (id == null || id.isEmpty()) { hdfsOperations.writeStringToFile(path, launcherConf, applicationIdStr); } else if (!applicationIdStr.equals(id)) { throw new LauncherException(MessageFormat.format( "YARN Id mismatch, action file [{0}] declares Id [{1}] current Id [{2}]", path, id, applicationIdStr)); } } } catch (RuntimeException | InterruptedException | IOException ex) { throw new LauncherException("IO error", ex); } }
From source file:org.apache.oozie.action.hadoop.TestYarnApplicationIdFinder.java
License:Apache License
@Test public void whenOldLauncherAndMRobApplicationsAreFinishedAndNewLauncherPresentNewLauncherIsUsed() throws Exception { final ApplicationReport oldLauncher = mock(ApplicationReport.class); when(oldLauncher.getApplicationType()).thenReturn("Oozie Launcher"); final ApplicationReport oldMRJob = mock(ApplicationReport.class); when(oldMRJob.getApplicationType()).thenReturn("MAPREDUCE"); final ApplicationId oldMRJobId = mock(ApplicationId.class); when(oldMRJobId.toString()).thenReturn("application_1534164756526_0002"); when(oldMRJob.getApplicationId()).thenReturn(oldMRJobId); final ApplicationReport newLauncher = mock(ApplicationReport.class); when(newLauncher.getApplicationType()).thenReturn("Oozie Launcher"); final ApplicationReport newMRJob = mock(ApplicationReport.class); when(newMRJob.getApplicationType()).thenReturn("MAPREDUCE"); final ApplicationId newMRJobId = mock(ApplicationId.class); when(newMRJobId.toString()).thenReturn("application_1534164756526_0004"); when(newMRJob.getApplicationId()).thenReturn(newMRJobId); when(reader.read())//from w ww . ja v a 2s.co m .thenReturn(new ArrayList<>(Arrays.asList(oldLauncher, oldMRJob, newLauncher, newMRJob))); when(workflowActionBean.getExternalId()).thenReturn("application_1534164756526_0003"); assertEquals("newLauncher should be found", "application_1534164756526_0004", yarnApplicationIdFinder.find()); when(workflowActionBean.getExternalId()).thenReturn("application_1534164756526_0004"); assertEquals("newLauncher should be found", "application_1534164756526_0004", yarnApplicationIdFinder.find()); when(workflowActionBean.getExternalId()).thenReturn("application_1534164756526_0005"); assertEquals("workflowActionBean.externalId should be found", "application_1534164756526_0005", yarnApplicationIdFinder.find()); }
From source file:org.apache.oozie.action.hadoop.TestYarnApplicationIdFinder.java
License:Apache License
@Test public void testGetLastYarnIdFromUnorderedListSuccess() { final ApplicationReport newLauncher = mock(ApplicationReport.class); final ApplicationId newLauncherId = mock(ApplicationId.class); when(newLauncherId.toString()).thenReturn("application_1534164756526_0003"); when(newLauncher.getApplicationId()).thenReturn(newLauncherId); final ApplicationReport newMRJob = mock(ApplicationReport.class); final ApplicationId newMRJobId = mock(ApplicationId.class); when(newMRJobId.toString()).thenReturn("application_1534164756526_0004"); when(newMRJob.getApplicationId()).thenReturn(newMRJobId); final String lastYarnId = yarnApplicationIdFinder.getLastYarnId(Arrays.asList(newMRJob, newLauncher)); assertEquals("last YARN id should be the maximal element in the list", "application_1534164756526_0004", lastYarnId);//from w w w . ja v a 2 s.c o m }
From source file:org.apache.oozie.tools.diag.OozieLauncherLogFetcher.java
License:Apache License
public static Path getRemoteAppLogDir(Path remoteRootLogDir, ApplicationId appId, String user, String suffix) { return new Path(getRemoteLogSuffixedDir(remoteRootLogDir, user, suffix), appId.toString()); }
From source file:org.apache.samza.validation.YarnJobValidationTool.java
License:Apache License
public ApplicationId validateAppId() throws Exception { // fetch only the last created application with the job name and id // i.e. get the application with max appId ApplicationId appId = null; for (ApplicationReport applicationReport : this.client.getApplications()) { if (applicationReport.getName().equals(this.jobName)) { ApplicationId id = applicationReport.getApplicationId(); if (appId == null || appId.compareTo(id) < 0) { appId = id;//from ww w.j a v a 2s . co m } } } if (appId != null) { log.info("Job lookup success. ApplicationId " + appId.toString()); return appId; } else { throw new SamzaException("Job lookup failure " + this.jobName); } }
From source file:org.apache.slider.client.SliderClient.java
License:Apache License
/** * * @param clustername name of the cluster * @param clusterDirectory cluster dir/*from ww w. ja v a2 s . com*/ * @param instanceDefinition the instance definition * @param debugAM enable debug AM options * @return the launched application * @throws YarnException * @throws IOException */ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException { deployedClusterName = clustername; SliderUtils.validateClusterName(clustername); verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config); //create the Slider AM provider -this helps set up the AM SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); instanceDefinition.resolve(); launchedInstanceDefinition = instanceDefinition; ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); MapOperations internalOptions = internalOperations.getGlobalOptions(); ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); Path generatedConfDirPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_GENERATED_CONF_PATH)); Path snapshotConfPath = createPathThatMustExist( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)); // cluster Provider AbstractClientProvider provider = createClientProvider( internalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME)); // make sure the conf dir is valid; if (log.isDebugEnabled()) { log.debug(instanceDefinition.toString()); } MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); // add the tags if available Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem, appOperations.getGlobalOptions().get(AgentKeys.APP_DEF)); AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config, sliderFileSystem, yarnClient, clusterSecure, sliderAMResourceComponent, resourceGlobalOptions, applicationTags); ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); amLauncher.setMaxAppAttempts(maxAppAttempts); sliderFileSystem.purgeAppInstanceTempFiles(clustername); Path tempPath = sliderFileSystem.createAppInstanceTempPath(clustername, appId.toString() + "/am"); String libdir = "lib"; Path libPath = new Path(tempPath, libdir); sliderFileSystem.getFileSystem().mkdirs(libPath); log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(), tempPath, libPath); // set local resources for the application master // local files or archives as needed // In this scenario, the jar file for the application master is part of the local resources Map<String, LocalResource> localResources = amLauncher.getLocalResources(); // look for the configuration directory named on the command line boolean hasServerLog4jProperties = false; Path remoteConfPath = null; String relativeConfDir = null; String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); if (confdirProp == null || confdirProp.isEmpty()) { log.debug("No local configuration directory provided as system property"); } else { File confDir = new File(confdirProp); if (!confDir.exists()) { throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir); } Path localConfDirPath = SliderUtils.createLocalPath(confDir); remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath); SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null); File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); hasServerLog4jProperties = log4jserver.isFile(); } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself boolean usingMiniMRCluster = getUsingMiniMRCluster(); if (!usingMiniMRCluster) { log.debug("Destination is not a MiniYARNCluster -copying full classpath"); // insert conf dir first if (remoteConfPath != null) { relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath, relativeConfDir); SliderUtils.mergeMaps(localResources, submittedConfDir); } } // build up the configuration // IMPORTANT: it is only after this call that site configurations // will be valid. propagatePrincipals(config, instanceDefinition); // validate security data /* // turned off until tested SecurityConfiguration securityConfiguration = new SecurityConfiguration(config, instanceDefinition, clustername); */ Configuration clientConfExtras = new Configuration(false); // then build up the generated path. FsPermission clusterPerms = getClusterDirectoryPermissions(config); SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms); // standard AM resources sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); //add provider-specific resources provider.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition, snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster); // now that the site config is fully generated, the provider gets // to do a quick review of them. log.debug("Preflight validation of cluster configuration"); sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); provider.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition, clusterDirectory, generatedConfDirPath, clusterSecure); // TODO: consider supporting apps that don't have an image path Path imagePath = SliderUtils.extractImagePath(sliderFileSystem, internalOptions); if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { log.debug("Registered image path {}", imagePath); } // build the environment amLauncher.putEnv(SliderUtils.buildEnvMap(sliderAMResourceComponent)); ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(), usingMiniMRCluster); amLauncher.setClasspath(classpath); if (log.isDebugEnabled()) { log.debug("AM classpath={}", classpath); log.debug("Environment Map:\n{}", SliderUtils.stringifyMap(amLauncher.getEnv())); log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); } // rm address InetSocketAddress rmSchedulerAddress; try { rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config); } catch (IllegalArgumentException e) { throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS, config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); } String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder(); // insert any JVM options); sliderAM.addJVMOptions(instanceDefinition, commandLine); // enable asserts if the text option is set commandLine.enableJavaAssertions(); // if the conf dir has a log4j-server.properties, switch to that if (hasServerLog4jProperties) { commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); } // add the AM sevice entry point commandLine.add(SliderAppMaster.SERVICE_CLASSNAME); // create action and the cluster name commandLine.add(ACTION_CREATE, clustername); // debug if (debugAM) { commandLine.add(Arguments.ARG_DEBUG); } // set the cluster directory path commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri()); if (!isUnset(rmAddr)) { commandLine.add(Arguments.ARG_RM_ADDR, rmAddr); } if (serviceArgs.getFilesystemBinding() != null) { commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); } /** * pass the registry binding */ addConfOptionToCLI(commandLine, config, REGISTRY_PATH, DEFAULT_REGISTRY_PATH); addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); if (clusterSecure) { // if the cluster is secure, make sure that // the relevant security settings go over /* addConfOptionToCLI(commandLine, config, KEY_SECURITY); */ addConfOptionToCLI(commandLine, config, DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); } // write out the path output commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); String cmdStr = commandLine.build(); log.debug("Completed setting up app master command {}", cmdStr); amLauncher.addCommandLine(commandLine); // the Slider AM gets to configure the AM requirements, not the custom provider sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, amLauncher.getResource()); // Set the priority for the application master int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY); amLauncher.setPriority(amPriority); // Set the queue to which this application is to be submitted in the RM // Queue for App master String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE); if (!SliderUtils.isUnset(suppliedQueue)) { amQueue = suppliedQueue; log.info("Using queue {} for the application instance.", amQueue); } if (amQueue != null) { amLauncher.setQueue(amQueue); } // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; }
From source file:org.apache.slider.server.appmaster.SliderAppMaster.java
License:Apache License
/** * Create and run the cluster./* w w w. ja v a 2s.c om*/ * @return exit code * @throws Throwable on a failure */ private int createAndRunCluster(String clustername) throws Throwable { //load the cluster description from the cd argument String sliderClusterDir = serviceArgs.getSliderClusterURI(); URI sliderClusterURI = new URI(sliderClusterDir); Path clusterDirPath = new Path(sliderClusterURI); log.info("Application defined at {}", sliderClusterURI); SliderFileSystem fs = getClusterFS(); // build up information about the running application -this // will be passed down to the cluster status MapOperations appInformation = new MapOperations(); AggregateConf instanceDefinition = InstanceIO.loadInstanceDefinitionUnresolved(fs, clusterDirPath); instanceDefinition.setName(clustername); log.info("Deploying cluster {}:", instanceDefinition); stateForProviders.setApplicationName(clustername); Configuration serviceConf = getConfig(); SecurityConfiguration securityConfiguration = new SecurityConfiguration(serviceConf, instanceDefinition, clustername); // obtain security state boolean securityEnabled = securityConfiguration.isSecurityEnabled(); // set the global security flag for the instance definition instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled); // triggers resolution and snapshotting in agent appState.updateInstanceDefinition(instanceDefinition); File confDir = getLocalConfDir(); if (!confDir.exists() || !confDir.isDirectory()) { log.info("Conf dir {} does not exist.", confDir); File parentFile = confDir.getParentFile(); log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile)); } // IP filtering serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME); //get our provider MapOperations globalInternalOptions = getGlobalInternalOptions(); String providerType = globalInternalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME); log.info("Cluster provider type is {}", providerType); SliderProviderFactory factory = SliderProviderFactory.createSliderProviderFactory(providerType); providerService = factory.createServerProvider(); // init the provider BUT DO NOT START IT YET initAndAddService(providerService); providerRMOperationHandler = new ProviderNotifyingOperationHandler(providerService); // create a slider AM provider sliderAMProvider = new SliderAMProviderService(); initAndAddService(sliderAMProvider); InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf); log.info("RM is at {}", address); yarnRPC = YarnRPC.create(serviceConf); /* * Extract the container ID. This is then * turned into an (incompete) container */ appMasterContainerID = ConverterUtils.toContainerId( SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name())); appAttemptID = appMasterContainerID.getApplicationAttemptId(); ApplicationId appid = appAttemptID.getApplicationId(); log.info("AM for ID {}", appid.getId()); appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString()); appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString()); appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString()); Map<String, String> envVars; List<Container> liveContainers; /** * It is critical this section is synchronized, to stop async AM events * arriving while registering a restarting AM. */ synchronized (appState) { int heartbeatInterval = HEARTBEAT_INTERVAL; //add the RM client -this brings the callbacks in asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, this); addService(asyncRMClient); //now bring it up deployChildService(asyncRMClient); //nmclient relays callbacks back to this class nmClientAsync = new NMClientAsyncImpl("nmclient", this); deployChildService(nmClientAsync); // set up secret manager secretManager = new ClientToAMTokenSecretManager(appAttemptID, null); if (securityEnabled) { // fix up the ACLs if they are not set String acls = getConfig().get(SliderXmlConfKeys.KEY_PROTOCOL_ACL); if (acls == null) { getConfig().set(SliderXmlConfKeys.KEY_PROTOCOL_ACL, "*"); } } //bring up the Slider RPC service startSliderRPCServer(instanceDefinition); rpcServiceAddress = rpcService.getConnectAddress(); appMasterHostname = rpcServiceAddress.getHostName(); appMasterRpcPort = rpcServiceAddress.getPort(); appMasterTrackingUrl = null; log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort); appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname); appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort); log.info("Starting Yarn registry"); registryOperations = startRegistryOperationsService(); log.info(registryOperations.toString()); //build the role map List<ProviderRole> providerRoles = new ArrayList<ProviderRole>(providerService.getRoles()); providerRoles.addAll(SliderAMClientProvider.ROLES); // Start up the WebApp and track the URL for it certificateManager = new CertificateManager(); MapOperations component = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM); certificateManager.initialize(component); certificateManager.setPassphrase(instanceDefinition.getPassphrase()); if (component.getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) { uploadServerCertForLocalization(clustername, fs); } startAgentWebApp(appInformation, serviceConf); int port = getPortToRequest(instanceDefinition); webApp = new SliderAMWebApp(registryOperations); WebApps.$for(SliderAMWebApp.BASE_PATH, WebAppApi.class, new WebAppApiImpl(this, stateForProviders, providerService, certificateManager, registryOperations), RestPaths.WS_CONTEXT).withHttpPolicy(serviceConf, HttpConfig.Policy.HTTP_ONLY).at(port) .start(webApp); String scheme = WebAppUtils.HTTP_PREFIX; appMasterTrackingUrl = scheme + appMasterHostname + ":" + webApp.port(); WebAppService<SliderAMWebApp> webAppService = new WebAppService<SliderAMWebApp>("slider", webApp); webAppService.init(serviceConf); webAppService.start(); addService(webAppService); appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/"); appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webApp.port()); // Register self with ResourceManager // This will start heartbeating to the RM // address = SliderUtils.getRmSchedulerAddress(asyncRMClient.getConfig()); log.info("Connecting to RM at {},address tracking URL={}", appMasterRpcPort, appMasterTrackingUrl); amRegistrationData = asyncRMClient.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); Resource maxResources = amRegistrationData.getMaximumResourceCapability(); containerMaxMemory = maxResources.getMemory(); containerMaxCores = maxResources.getVirtualCores(); appState.setContainerLimits(maxResources.getMemory(), maxResources.getVirtualCores()); // build the handler for RM request/release operations; this uses // the max value as part of its lookup rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maxResources); // set the RM-defined maximum cluster values appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores)); appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory)); // process the initial user to obtain the set of user // supplied credentials (tokens were passed in by client). Remove AMRM // token and HDFS delegation token, the latter because we will provide an // up to date token for container launches (getContainerCredentials()). UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); Credentials credentials = currentUser.getCredentials(); Iterator<Token<? extends TokenIdentifier>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token<? extends TokenIdentifier> token = iter.next(); log.info("Token {}", token.getKind()); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME) || token.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) { iter.remove(); } } // at this point this credentials map is probably clear, but leaving this // code to allow for future tokens... containerCredentials = credentials; if (securityEnabled) { secretManager.setMasterKey(amRegistrationData.getClientToAMTokenMasterKey().array()); applicationACLs = amRegistrationData.getApplicationACLs(); //tell the server what the ACLs are rpcService.getServer().refreshServiceAcl(serviceConf, new SliderAMPolicyProvider()); // perform keytab based login to establish kerberos authenticated // principal. Can do so now since AM registration with RM above required // tokens associated to principal String principal = securityConfiguration.getPrincipal(); File localKeytabFile = securityConfiguration.getKeytabFile(instanceDefinition); // Now log in... login(principal, localKeytabFile); // obtain new FS reference that should be kerberos based and different // than the previously cached reference fs = getClusterFS(); } // extract container list liveContainers = amRegistrationData.getContainersFromPreviousAttempts(); //now validate the installation Configuration providerConf = providerService.loadProviderConfigurationInformation(confDir); providerService.initializeApplicationConfiguration(instanceDefinition, fs); providerService.validateApplicationConfiguration(instanceDefinition, confDir, securityEnabled); //determine the location for the role history data Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME); //build the instance appState.buildInstance(instanceDefinition, serviceConf, providerConf, providerRoles, fs.getFileSystem(), historyDir, liveContainers, appInformation, new SimpleReleaseSelector()); providerService.rebuildContainerDetails(liveContainers, instanceDefinition.getName(), appState.getRolePriorityMap()); // add the AM to the list of nodes in the cluster appState.buildAppMasterNode(appMasterContainerID, appMasterHostname, webApp.port(), appMasterHostname + ":" + webApp.port()); // build up environment variables that the AM wants set in every container // irrespective of provider and role. envVars = new HashMap<String, String>(); if (hadoop_user_name != null) { envVars.put(HADOOP_USER_NAME, hadoop_user_name); } } String rolesTmpSubdir = appMasterContainerID.toString() + "/roles"; String amTmpDir = globalInternalOptions.getMandatoryOption(InternalKeys.INTERNAL_AM_TMP_DIR); Path tmpDirPath = new Path(amTmpDir); Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir); fs.getFileSystem().mkdirs(launcherTmpDirPath); //launcher service launchService = new RoleLaunchService(actionQueues, providerService, fs, new Path(getGeneratedConfDir()), envVars, launcherTmpDirPath); deployChildService(launchService); appState.noteAMLaunched(); //Give the provider access to the state, and AM providerService.bind(stateForProviders, actionQueues, liveContainers); sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers); // chaos monkey maybeStartMonkey(); // setup token renewal and expiry handling for long lived apps // if (SliderUtils.isHadoopClusterSecure(getConfig())) { // fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues); // fsDelegationTokenManager.acquireDelegationToken(getConfig()); // } // if not a secure cluster, extract the username -it will be // propagated to workers if (!UserGroupInformation.isSecurityEnabled()) { hadoop_user_name = System.getenv(HADOOP_USER_NAME); log.info(HADOOP_USER_NAME + "='{}'", hadoop_user_name); } service_user_name = RegistryUtils.currentUser(); log.info("Registry service username ={}", service_user_name); // now do the registration registerServiceInstance(clustername, appid); // log the YARN and web UIs log.info("RM Webapp address {}", serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS)); log.info("slider Webapp address {}", appMasterTrackingUrl); // declare the cluster initialized log.info("Application Master Initialization Completed"); initCompleted.set(true); try { // start handling any scheduled events startQueueProcessing(); // Start the Slider AM provider sliderAMProvider.start(); // launch the real provider; this is expected to trigger a callback that // starts the node review process launchProviderService(instanceDefinition, confDir); //now block waiting to be told to exit the process waitForAMCompletionSignal(); } catch (Exception e) { log.error("Exception : {}", e, e); onAMStop(new ActionStopSlider(e)); } //shutdown time return finish(); }
From source file:org.apache.slider.server.appmaster.SliderAppMaster.java
License:Apache License
/** * This registers the service instance and its external values * @param instanceName name of this instance * @param appid application ID//from w ww. j a v a2 s. com * @throws IOException */ private void registerServiceInstance(String instanceName, ApplicationId appid) throws IOException { // the registry is running, so register services URL amWebURI = new URL(appMasterTrackingUrl); URL agentOpsURI = new URL(agentOpsUrl); URL agentStatusURI = new URL(agentStatusUrl); //Give the provider restricted access to the state, registry setupInitialRegistryPaths(); yarnRegistryOperations = new YarnRegistryViewForProviders(registryOperations, service_user_name, SliderKeys.APP_TYPE, instanceName, appAttemptID); providerService.bindToYarnRegistry(yarnRegistryOperations); sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations); // Yarn registry ServiceRecord serviceRecord = new ServiceRecord(); serviceRecord.set(YarnRegistryAttributes.YARN_ID, appid.toString()); serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.APPLICATION); serviceRecord.description = "Slider Application Master"; /* SLIDER-531: disable this addition so things compile against versions of the registry with/without the new record format serviceRecord.addExternalEndpoint( RegistryTypeUtils.ipcEndpoint( CustomRegistryConstants.AM_IPC_PROTOCOL, rpcServiceAddress)); */ // internal services sliderAMProvider.applyInitialRegistryDefinitions(amWebURI, agentOpsURI, agentStatusURI, serviceRecord); // provider service dynamic definitions. providerService.applyInitialRegistryDefinitions(amWebURI, agentOpsURI, agentStatusURI, serviceRecord); // register the service's entry log.info("Service Record \n{}", serviceRecord); yarnRegistryOperations.registerSelf(serviceRecord, true); log.info("Registered service under {}; absolute path {}", yarnRegistryOperations.getSelfRegistrationPath(), yarnRegistryOperations.getAbsoluteSelfRegistrationPath()); boolean isFirstAttempt = 1 == appAttemptID.getAttemptId(); // delete the children in case there are any and this is an AM startup. // just to make sure everything underneath is purged if (isFirstAttempt) { yarnRegistryOperations.deleteChildren(yarnRegistryOperations.getSelfRegistrationPath(), true); } }
From source file:org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin.java
License:Apache License
private Set<TimelineEntityGroupId> toGroupId(ApplicationId applicationId) { TimelineEntityGroupId groupId = TimelineEntityGroupId.newInstance(applicationId, applicationId.toString()); LOG.debug("mapped {} to {}, ", applicationId, groupId); Set<TimelineEntityGroupId> result = new HashSet<>(); result.add(groupId);//from w w w. j a va 2 s . c o m return result; }