Example usage for org.apache.hadoop.registry.client.api RegistryConstants KEY_REGISTRY_ZK_QUORUM

Introduction

In this page you can find the example usage for org.apache.hadoop.registry.client.api RegistryConstants KEY_REGISTRY_ZK_QUORUM.

Prototype

String KEY_REGISTRY_ZK_QUORUM

To view the source code for org.apache.hadoop.registry.client.api RegistryConstants KEY_REGISTRY_ZK_QUORUM.

Click Source Link

Document

List of hostname:port pairs defining the zookeeper quorum binding for the registry .

Usage

From source file:org.apache.slider.client.SliderClient.java

License:Apache License

/**
 *
 * @param clustername name of the cluster
 * @param clusterDirectory cluster dir/*from   w  w  w  . ja v a2  s. c  o  m*/
 * @param instanceDefinition the instance definition
 * @param debugAM enable debug AM options
 * @return the launched application
 * @throws YarnException
 * @throws IOException
 */
public LaunchedApplication launchApplication(String clustername, Path clusterDirectory,
        AggregateConf instanceDefinition, boolean debugAM) throws YarnException, IOException {

    deployedClusterName = clustername;
    SliderUtils.validateClusterName(clustername);
    verifyNoLiveClusters(clustername, "Launch");
    Configuration config = getConfig();
    lookupZKQuorum();
    boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
    //create the Slider AM provider -this helps set up the AM
    SliderAMClientProvider sliderAM = new SliderAMClientProvider(config);

    instanceDefinition.resolve();
    launchedInstanceDefinition = instanceDefinition;

    ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations();
    MapOperations internalOptions = internalOperations.getGlobalOptions();
    ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations();
    ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations();
    Path generatedConfDirPath = createPathThatMustExist(
            internalOptions.getMandatoryOption(InternalKeys.INTERNAL_GENERATED_CONF_PATH));
    Path snapshotConfPath = createPathThatMustExist(
            internalOptions.getMandatoryOption(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));

    // cluster Provider
    AbstractClientProvider provider = createClientProvider(
            internalOptions.getMandatoryOption(InternalKeys.INTERNAL_PROVIDER_NAME));
    // make sure the conf dir is valid;

    if (log.isDebugEnabled()) {
        log.debug(instanceDefinition.toString());
    }
    MapOperations sliderAMResourceComponent = resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
    MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions();

    // add the tags if available
    Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
            appOperations.getGlobalOptions().get(AgentKeys.APP_DEF));
    AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, SliderKeys.APP_TYPE, config,
            sliderFileSystem, yarnClient, clusterSecure, sliderAMResourceComponent, resourceGlobalOptions,
            applicationTags);

    ApplicationId appId = amLauncher.getApplicationId();
    // set the application name;
    amLauncher.setKeepContainersOverRestarts(true);

    int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
    amLauncher.setMaxAppAttempts(maxAppAttempts);

    sliderFileSystem.purgeAppInstanceTempFiles(clustername);
    Path tempPath = sliderFileSystem.createAppInstanceTempPath(clustername, appId.toString() + "/am");
    String libdir = "lib";
    Path libPath = new Path(tempPath, libdir);
    sliderFileSystem.getFileSystem().mkdirs(libPath);
    log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem.toString(), tempPath, libPath);
    // set local resources for the application master
    // local files or archives as needed
    // In this scenario, the jar file for the application master is part of the local resources
    Map<String, LocalResource> localResources = amLauncher.getLocalResources();

    // look for the configuration directory named on the command line
    boolean hasServerLog4jProperties = false;
    Path remoteConfPath = null;
    String relativeConfDir = null;
    String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
    if (confdirProp == null || confdirProp.isEmpty()) {
        log.debug("No local configuration directory provided as system property");
    } else {
        File confDir = new File(confdirProp);
        if (!confDir.exists()) {
            throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, confDir);
        }
        Path localConfDirPath = SliderUtils.createLocalPath(confDir);
        remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
        log.debug("Slider configuration directory is {}; remote to be {}", localConfDirPath, remoteConfPath);
        SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);

        File log4jserver = new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
        hasServerLog4jProperties = log4jserver.isFile();
    }
    // the assumption here is that minimr cluster => this is a test run
    // and the classpath can look after itself

    boolean usingMiniMRCluster = getUsingMiniMRCluster();
    if (!usingMiniMRCluster) {

        log.debug("Destination is not a MiniYARNCluster -copying full classpath");

        // insert conf dir first
        if (remoteConfPath != null) {
            relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR;
            Map<String, LocalResource> submittedConfDir = sliderFileSystem.submitDirectory(remoteConfPath,
                    relativeConfDir);
            SliderUtils.mergeMaps(localResources, submittedConfDir);
        }
    }
    // build up the configuration 
    // IMPORTANT: it is only after this call that site configurations
    // will be valid.

    propagatePrincipals(config, instanceDefinition);
    // validate security data

    /*
        // turned off until tested
        SecurityConfiguration securityConfiguration =
            new SecurityConfiguration(config,
    instanceDefinition, clustername);
                
    */
    Configuration clientConfExtras = new Configuration(false);
    // then build up the generated path.
    FsPermission clusterPerms = getClusterDirectoryPermissions(config);
    SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath, clusterPerms);

    // standard AM resources
    sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition,
            snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster);
    //add provider-specific resources
    provider.prepareAMAndConfigForLaunch(sliderFileSystem, config, amLauncher, instanceDefinition,
            snapshotConfPath, generatedConfDirPath, clientConfExtras, libdir, tempPath, usingMiniMRCluster);

    // now that the site config is fully generated, the provider gets
    // to do a quick review of them.
    log.debug("Preflight validation of cluster configuration");

    sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition,
            clusterDirectory, generatedConfDirPath, clusterSecure);

    provider.preflightValidateClusterConfiguration(sliderFileSystem, clustername, config, instanceDefinition,
            clusterDirectory, generatedConfDirPath, clusterSecure);

    // TODO: consider supporting apps that don't have an image path
    Path imagePath = SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
    if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
        log.debug("Registered image path {}", imagePath);
    }

    // build the environment
    amLauncher.putEnv(SliderUtils.buildEnvMap(sliderAMResourceComponent));
    ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir, libdir, getConfig(),
            usingMiniMRCluster);
    amLauncher.setClasspath(classpath);
    if (log.isDebugEnabled()) {
        log.debug("AM classpath={}", classpath);
        log.debug("Environment Map:\n{}", SliderUtils.stringifyMap(amLauncher.getEnv()));
        log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
    }

    // rm address

    InetSocketAddress rmSchedulerAddress;
    try {
        rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config);
    } catch (IllegalArgumentException e) {
        throw new BadConfigException("%s Address invalid: %s", YarnConfiguration.RM_SCHEDULER_ADDRESS,
                config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));

    }
    String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);

    JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder();
    // insert any JVM options);
    sliderAM.addJVMOptions(instanceDefinition, commandLine);
    // enable asserts if the text option is set
    commandLine.enableJavaAssertions();

    // if the conf dir has a log4j-server.properties, switch to that
    if (hasServerLog4jProperties) {
        commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
        commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    }

    // add the AM sevice entry point
    commandLine.add(SliderAppMaster.SERVICE_CLASSNAME);

    // create action and the cluster name
    commandLine.add(ACTION_CREATE, clustername);

    // debug
    if (debugAM) {
        commandLine.add(Arguments.ARG_DEBUG);
    }

    // set the cluster directory path
    commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri());

    if (!isUnset(rmAddr)) {
        commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
    }

    if (serviceArgs.getFilesystemBinding() != null) {
        commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
    }

    /**
     * pass the registry binding
     */
    addConfOptionToCLI(commandLine, config, REGISTRY_PATH, DEFAULT_REGISTRY_PATH);
    addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);

    if (clusterSecure) {
        // if the cluster is secure, make sure that
        // the relevant security settings go over
        /*
              addConfOptionToCLI(commandLine, config, KEY_SECURITY);
        */
        addConfOptionToCLI(commandLine, config, DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
    }
    // write out the path output
    commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);

    String cmdStr = commandLine.build();
    log.debug("Completed setting up app master command {}", cmdStr);

    amLauncher.addCommandLine(commandLine);

    // the Slider AM gets to configure the AM requirements, not the custom provider
    sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, amLauncher.getResource());

    // Set the priority for the application master

    int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY, DEFAULT_YARN_QUEUE_PRIORITY);

    amLauncher.setPriority(amPriority);

    // Set the queue to which this application is to be submitted in the RM
    // Queue for App master
    String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
    String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
    if (!SliderUtils.isUnset(suppliedQueue)) {
        amQueue = suppliedQueue;
        log.info("Using queue {} for the application instance.", amQueue);
    }

    if (amQueue != null) {
        amLauncher.setQueue(amQueue);
    }

    // submit the application
    LaunchedApplication launchedApplication = amLauncher.submitApplication();
    return launchedApplication;
}

---

From source file:org.apache.slider.common.tools.ConfigHelper.java

License:Apache License

/**
 * Register anything we consider deprecated
 *///from   www  .java 2 s  .c  o m
public static void registerDeprecatedConfigItems() {
    Configuration.addDeprecation(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
            RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
    Configuration.addDeprecation(SliderXmlConfKeys.REGISTRY_PATH, RegistryConstants.KEY_REGISTRY_ZK_ROOT);

}

---

From source file:org.apache.slider.server.services.utility.AbstractSliderLaunchedService.java

License:Apache License

/**
 * look up the registry quorum from the config
 * @return the quorum string//www.  j a  v  a  2  s. c om
 * @throws BadConfigException if it is not there or invalid
 */
public String lookupZKQuorum() throws BadConfigException {

    String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);

    // though if neither is set: trouble
    if (SliderUtils.isUnset(registryQuorum)) {
        throw new BadConfigException("No Zookeeper quorum provided in the" + " configuration property "
                + RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
    }
    ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
    return registryQuorum;
}

---