Example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_ADDRESS

List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_ADDRESS

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.yarn.conf.*
  5. YarnConfiguration
  6. RM_SCHEDULER_ADDRESS

Introduction

In this page you can find the example usage for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_ADDRESS.

Prototype

String RM_SCHEDULER_ADDRESS

To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER_ADDRESS.

Click Source Link

Document

The address of the scheduler interface.

Usage

From source file:com.cloudera.kitten.appmaster.service.ResourceManagerConnectionFactory.java

License:Open Source License

@Override
public AMRMProtocol connect() {
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    return ((AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf));
}

From source file:com.continuuity.weave.internal.yarn.ports.AMRMClientImpl.java

License:Apache License

@Override
public synchronized void start() {
    final YarnConfiguration conf = new YarnConfiguration(getConfig());
    final YarnRPC rpc = YarnRPC.create(conf);
    final InetSocketAddress rmAddress = conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

    UserGroupInformation currentUser;/*from w  ww .j  a  va 2s.c  o  m*/
    try {
        currentUser = UserGroupInformation.getCurrentUser();
    } catch (IOException e) {
        throw new YarnException(e);
    }

    if (UserGroupInformation.isSecurityEnabled()) {
        String tokenURLEncodedStr = System.getenv().get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
        Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();

        try {
            token.decodeFromUrlString(tokenURLEncodedStr);
        } catch (IOException e) {
            throw new YarnException(e);
        }

        SecurityUtil.setTokenService(token, rmAddress);
        if (LOG.isDebugEnabled()) {
            LOG.debug("AppMasterToken is " + token);
        }
        currentUser.addToken(token);
    }

    rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
        @Override
        public AMRMProtocol run() {
            return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf);
        }
    });
    LOG.debug("Connecting to ResourceManager at " + rmAddress);
    super.start();
}

From source file:com.github.sakserv.minicluster.impl.MRLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {

    // Handle Windows
    WindowsLibsUtils.setHadoopHome();/*from w w  w  . j  a  va  2 s.c  o  m*/

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(JHAdminConfig.MR_HISTORY_ADDRESS, jobHistoryAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    configuration.set(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }

    if (null != hdfsDefaultFs) {
        configuration.set("fs.defaultFS", hdfsDefaultFs);
        configuration.set("dfs.replication", "1");
    }
}

From source file:com.github.sakserv.minicluster.impl.YarnLocalCluster.java

License:Apache License

@Override
public void configure() throws Exception {
    // Handle Windows
    WindowsLibsUtils.setHadoopHome();//from   w w w.ja va2s  . c o m

    configuration.set(YarnConfiguration.RM_ADDRESS, resourceManagerAddress);
    configuration.set(YarnConfiguration.RM_HOSTNAME, resourceManagerHostname);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resourceManagerSchedulerAddress);
    configuration.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resourceManagerResourceTrackerAddress);
    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resourceManagerWebappAddress);
    configuration.set(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, "true");
    if (getUseInJvmContainerExecutor()) {
        configuration.set(YarnConfiguration.NM_CONTAINER_EXECUTOR, inJvmContainerExecutorClass);
        configuration.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
        configuration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
    }
}

From source file:com.toy.Client.java

License:Apache License

/**
 * Start a new Application Master and deploy the web application on 2 Tomcat containers
 *
 * @throws Exception/*from  w  w w.  j  a va2 s . com*/
 */
void start() throws Exception {

    //Check tomcat dir
    final File tomcatHomeDir = new File(toyConfig.tomcat);
    final File tomcatLibraries = new File(tomcatHomeDir, "lib");
    final File tomcatBinaries = new File(tomcatHomeDir, "bin");
    Preconditions.checkState(tomcatLibraries.isDirectory(),
            tomcatLibraries.getAbsolutePath() + " does not exist");

    //Check war file
    final File warFile = new File(toyConfig.war);
    Preconditions.checkState(warFile.isFile(), warFile.getAbsolutePath() + " does not exist");

    yarn = YarnClient.createYarnClient();
    yarn.init(configuration);
    yarn.start();

    YarnClientApplication yarnApplication = yarn.createApplication();
    GetNewApplicationResponse newApplication = yarnApplication.getNewApplicationResponse();
    appId = newApplication.getApplicationId();
    ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext();
    appContext.setApplicationName("Tomcat : " + tomcatHomeDir.getName() + "\n War : " + warFile.getName());
    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    // Register required libraries
    Map<String, LocalResource> localResources = new HashMap<>();
    FileSystem fs = FileSystem.get(configuration);
    uploadDepAndRegister(localResources, appId, fs, "lib-ext/curator-client-2.3.0.jar");
    uploadDepAndRegister(localResources, appId, fs, "lib-ext/curator-framework-2.3.0.jar");
    uploadDepAndRegister(localResources, appId, fs, "lib-ext/curator-recipes-2.3.0.jar");

    // Register application master jar
    registerLocalResource(localResources, appId, fs, new Path(appMasterJar));

    // Register the WAR that will be deployed on Tomcat
    registerLocalResource(localResources, appId, fs, new Path(warFile.getAbsolutePath()));

    // Register Tomcat libraries
    for (File lib : tomcatLibraries.listFiles()) {
        registerLocalResource(localResources, appId, fs, new Path(lib.getAbsolutePath()));
    }

    File juli = new File(tomcatBinaries, "tomcat-juli.jar");
    if (juli.exists()) {
        registerLocalResource(localResources, appId, fs, new Path(juli.getAbsolutePath()));
    }

    amContainer.setLocalResources(localResources);

    // Setup master environment
    Map<String, String> env = new HashMap<>();
    final String TOMCAT_LIBS = fs.getHomeDirectory() + "/" + Constants.TOY_PREFIX + appId.toString();
    env.put(Constants.TOMCAT_LIBS, TOMCAT_LIBS);

    if (toyConfig.zookeeper != null) {
        env.put(Constants.ZOOKEEPER_QUORUM, toyConfig.zookeeper);
    } else {
        env.put(Constants.ZOOKEEPER_QUORUM, NetUtils.getHostname());
    }

    // 1. Compute classpath
    StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$())
            .append(File.pathSeparatorChar).append("./*");
    for (String c : configuration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar);
        classPathEnv.append(c.trim());
    }
    classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");

    // add the runtime classpath needed for tests to work
    if (configuration.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(':');
        classPathEnv.append(System.getProperty("java.class.path"));
    }
    env.put("CLASSPATH", classPathEnv.toString());
    env.put(Constants.WAR, warFile.getName());
    // For unit test with YarnMiniCluster
    env.put(YarnConfiguration.RM_SCHEDULER_ADDRESS, configuration.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
    amContainer.setEnvironment(env);

    // 1.2 Set constraint for the app master
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(32);
    appContext.setResource(capability);

    // 2. Compute app master cmd line
    Vector<CharSequence> vargs = new Vector<>(10);
    // Set java executable command
    vargs.add(ApplicationConstants.Environment.JAVA_HOME.$() + "/bin/java");
    // Set Xmx based on am memory size
    vargs.add("-Xmx32m");
    // Set class name
    vargs.add(TOYMaster.class.getCanonicalName());
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");

    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
        command.append(str).append(" ");
    }

    LOG.info("Completed setting up app master command " + command.toString());
    List<String> commands = new ArrayList<>();
    commands.add(command.toString());
    amContainer.setCommands(commands);
    appContext.setAMContainerSpec(amContainer);

    // 3. Setup security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
        Credentials credentials = new Credentials();
        String tokenRenewer = configuration.get(YarnConfiguration.RM_PRINCIPAL);
        if (tokenRenewer == null || tokenRenewer.length() == 0) {
            throw new Exception("Can't get Master Kerberos principal for the RM to use as renewer");
        }

        // For now, only getting tokens for the default file-system.
        final org.apache.hadoop.security.token.Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer,
                credentials);
        if (tokens != null) {
            for (org.apache.hadoop.security.token.Token<?> token : tokens) {
                LOG.info("Got dt for " + fs.getUri() + "; " + token);
            }
        }
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        amContainer.setTokens(fsTokens);
    }

    appContext.setQueue("default");
    LOG.info("Submitting TOY application {} to ASM", appId.toString());
    yarn.submitApplication(appContext);

    // Monitor the application and exit if it is RUNNING
    monitorApplication(appId);
}

From source file:com.toy.TOYMaster.java

License:Apache License

public TOYMaster() throws Exception {
    this.configuration = new YarnConfiguration();
    final String rm_address = System.getenv(YarnConfiguration.RM_SCHEDULER_ADDRESS);
    LOG.info("RM is @ {}", rm_address);
    configuration.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rm_address);
    this.fs = FileSystem.get(configuration);
    init();/* w  ww.  j a v  a 2 s. c  o m*/
}

From source file:edu.uci.ics.hyracks.yarn.common.protocols.amrm.AMRMConnection.java

License:Apache License

public AMRMConnection(YarnConfiguration config) {
    this.config = config;
    Map<String, String> envs = System.getenv();
    String containerIdString = envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
    if (containerIdString == null) {
        throw new IllegalArgumentException("ContainerId not set in the environment");
    }//from   w  ww. j  a va 2 s  . c  om
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    appAttemptId = containerId.getApplicationAttemptId();
    InetSocketAddress rmAddress = NetUtils.createSocketAddr(
            config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
    YarnRPC rpc = YarnRPC.create(config);

    amrmp = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, config);
}

From source file:org.apache.hama.bsp.BSPApplicationMaster.java

License:Apache License

/**
 * Connects to the Resource Manager.//from  ww w . ja v a2  s  .  c om
 * 
 * @param yarnConf
 * @return a new RPC connection to the Resource Manager.
 */
private ApplicationMasterProtocol getYarnRPCConnection(Configuration yarnConf) throws IOException {
    // Connect to the Scheduler of the ResourceManager.
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(appAttemptId.toString());
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    final InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf
            .get(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));

    Token<? extends TokenIdentifier> amRMToken = setupAndReturnAMRMToken(rmAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);

    final Configuration conf = yarnConf;

    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) yarnRPC.getProxy(ApplicationMasterProtocol.class, rmAddress,
                    conf);
        }
    });
    LOG.info("Connecting to ResourceManager at " + rmAddress);
    return client;
}

From source file:org.apache.hoya.tools.HoyaUtils.java

License:Apache License

public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
    return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
            YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
}

From source file:org.apache.hoya.tools.HoyaUtils.java

License:Apache License

public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
    conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
}