Example usage for org.apache.hadoop.mapreduce MRConfig MASTER

Introduction

In this page you can find the example usage for org.apache.hadoop.mapreduce MRConfig MASTER_ADDRESS.

Prototype

String MASTER_ADDRESS

To view the source code for org.apache.hadoop.mapreduce MRConfig MASTER_ADDRESS.

Click Source Link

Usage

From source file:org.apache.ignite.client.hadoop.GridHadoopClientProtocolProvider.java

License:Apache License

/** {@inheritDoc} */
@Override// ww  w  .  j  a  v  a2 s  .  c  o  m
public ClientProtocol create(Configuration conf) throws IOException {
    if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME))) {
        String addr = conf.get(MRConfig.MASTER_ADDRESS);

        if (F.isEmpty(addr))
            throw new IOException(
                    "Failed to create client protocol because server address is not specified (is "
                            + MRConfig.MASTER_ADDRESS + " property set?).");

        if (F.eq(addr, "local"))
            throw new IOException("Local execution mode is not supported, please point "
                    + MRConfig.MASTER_ADDRESS + " to real Ignite node.");

        return createProtocol(addr, conf);
    }

    return null;
}

From source file:org.apache.ignite.client.hadoop.GridHadoopClientProtocolSelfTest.java

License:Apache License

/**
 * @return Configuration./*from  ww  w  . j a v a2  s.  co  m*/
 */
private Configuration config(int port) {
    Configuration conf = new Configuration();

    setupFileSystems(conf);

    conf.set(MRConfig.FRAMEWORK_NAME, GridHadoopClientProtocol.FRAMEWORK_NAME);
    conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + port);

    conf.set("fs.defaultFS", "igfs://:" + getTestGridName(0) + "@/");

    return conf;
}

From source file:org.apache.ignite.client.hadoop.HadoopClientProtocolSelfTest.java

License:Apache License

/**
 * @return Configuration.//  ww w . ja v  a  2  s  . co  m
 */
private Configuration config(int port) {
    Configuration conf = new Configuration();

    setupFileSystems(conf);

    conf.set(MRConfig.FRAMEWORK_NAME, IgniteHadoopClientProtocolProvider.FRAMEWORK_NAME);
    conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + port);

    conf.set("fs.defaultFS", "igfs://:" + getTestGridName(0) + "@/");

    return conf;
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolMultipleServersSelfTest.java

License:Apache License

/**
 * @return Configuration./*from  ww w .  jav  a2  s .com*/
 */
private Configuration configSingleAddress() {
    Configuration conf = HadoopUtils.safeCreateConfiguration();

    setupFileSystems(conf);

    conf.set(MRConfig.FRAMEWORK_NAME, IgniteHadoopClientProtocolProvider.FRAMEWORK_NAME);
    conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + REST_PORT);

    conf.set("fs.defaultFS", "igfs://" + igfsName + "@/");

    return conf;
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolMultipleServersSelfTest.java

License:Apache License

/**
 * @param srvsCnt Count ov servers./*from w  w  w.  j a  va2 s . c o  m*/
 * @return Configuration.
 */
private Configuration configMultipleAddrs(int srvsCnt) {
    Configuration conf = HadoopUtils.safeCreateConfiguration();

    setupFileSystems(conf);

    conf.set(MRConfig.FRAMEWORK_NAME, IgniteHadoopClientProtocolProvider.FRAMEWORK_NAME);

    Collection<String> addrs = new ArrayList<>(srvsCnt);

    for (int i = 0; i < srvsCnt; ++i)
        addrs.add("127.0.0.1:" + Integer.toString(REST_PORT + i));

    conf.set(MRConfig.MASTER_ADDRESS, F.concat(addrs, ","));

    conf.set("fs.defaultFS", "igfs://" + igfsName + "@/");

    return conf;
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolMultipleServersSelfTest.java

License:Apache License

/**
 * @return Configuration./*  w w w. j a  va2 s.  c o  m*/
 */
private Configuration configMixed() {
    Configuration conf = HadoopUtils.safeCreateConfiguration();

    setupFileSystems(conf);

    conf.set(MRConfig.FRAMEWORK_NAME, IgniteHadoopClientProtocolProvider.FRAMEWORK_NAME);

    Collection<String> addrs = new ArrayList<>();

    addrs.add("localhost");
    addrs.add("127.0.0.1:" + Integer.toString(REST_PORT + 1));

    conf.set(MRConfig.MASTER_ADDRESS, F.concat(addrs, ","));

    conf.set("fs.defaultFS", "igfs://" + igfsName + "@/");

    return conf;
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.client.HadoopClientProtocolSelfTest.java

License:Apache License

/**
 * @return Configuration./*w  w w .j a  va  2s .  c o m*/
 */
private Configuration config(int port) {
    Configuration conf = HadoopUtils.safeCreateConfiguration();

    setupFileSystems(conf);

    conf.set(MRConfig.FRAMEWORK_NAME, IgniteHadoopClientProtocolProvider.FRAMEWORK_NAME);
    conf.set(MRConfig.MASTER_ADDRESS, "127.0.0.1:" + port);

    conf.set("fs.defaultFS", "igfs://@/");

    return conf;
}

From source file:org.apache.solr.hadoop.hack.MiniMRYarnCluster.java

License:Apache License

@Override
public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath());
    }/*from  w  w  w .j a  v a  2  s  .  co m*/

    // By default, VMEM monitoring disabled, PMEM monitoring enabled.
    if (!conf.getBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
            MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
        conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
        conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
    }

    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
        Path stagingPath = FileContext.getFileContext(conf)
                .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        /*
         * Re-configure the staging path on Windows if the file system is localFs.
         * We need to use a absolute path that contains the drive letter. The unit
         * test could run on a different drive than the AM. We can run into the
         * issue that job files are localized to the drive where the test runs on,
         * while the AM starts on a different drive and fails to find the job
         * metafiles. Using absolute path can avoid this ambiguity.
         */
        if (Path.WINDOWS) {
            if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
                conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                        new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath());
            }
        }
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        //mkdir the staging directory so that right permissions are set while running as proxy user
        fc.mkdir(stagingPath, null, true);
        //mkdir done directory as well 
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new YarnRuntimeException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
                                               // which shuffle doesn't happen
                                               //configure the shuffle service in NM
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
            new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
    conf.setClass(String.format(Locale.ENGLISH, YarnConfiguration.NM_AUX_SERVICE_FMT,
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class);

    // Non-standard shuffle port
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class,
            ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

    super.serviceInit(conf);
}

From source file:org.apache.tajo.MiniTajoYarnCluster.java

License:Apache License

@Override
public void init(Configuration conf) {

    conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, new InetSocketAddress("127.0.0.1", 0));
    conf.setSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, new InetSocketAddress("127.0.0.1", 0));

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
                new File(getTestWorkDir(), "apps_staging_dir/").getAbsolutePath());
    }//from  www  .j a va2s .c o m
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
        Path stagingPath = FileContext.getFileContext(conf)
                .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
        FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
        if (fc.util().exists(stagingPath)) {
            LOG.info(stagingPath + " exists! deleting...");
            fc.delete(stagingPath, true);
        }
        LOG.info("mkdir: " + stagingPath);
        //mkdir the staging directory so that right permissions are set while running as proxy user
        fc.mkdir(stagingPath, null, true);
        //mkdir done directory as well
        String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
        Path doneDirPath = fc.makeQualified(new Path(doneDir));
        fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
        throw new YarnRuntimeException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
    // which shuffle doesn't happen
    //configure the shuffle service in NM
    conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, PullServerAuxService.PULLSERVER_SERVICEID);
    conf.setClass(
            String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, PullServerAuxService.PULLSERVER_SERVICEID),
            PullServerAuxService.class, Service.class);

    // Non-standard shuffle port
    conf.setInt(TajoConf.ConfVars.PULLSERVER_PORT.name(), 0);

    // local directory
    conf.set(TajoConf.ConfVars.WORKER_TEMPORAL_DIR.name(), "/tmp/tajo-localdir");

    conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR, DefaultContainerExecutor.class,
            ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

    conf.setInt("yarn.nodemanager.delete.debug-delay-sec", 600);

    super.init(conf);
}

From source file:org.gridgain.client.hadoop.GridHadoopClientProtocolProvider.java

License:Open Source License

/** {@inheritDoc} */
@Override//from   www  .ja  v  a 2  s. c om
public ClientProtocol create(Configuration conf) throws IOException {
    if (FRAMEWORK_NAME.equals(conf.get(MRConfig.FRAMEWORK_NAME))) {
        String addr = conf.get(MRConfig.MASTER_ADDRESS);

        if (F.isEmpty(addr))
            throw new IOException(
                    "Failed to create client protocol because server address is not specified (is "
                            + MRConfig.MASTER_ADDRESS + " property set?).");

        if (F.eq(addr, "local"))
            throw new IOException("Local execution mode is not supported, please point "
                    + MRConfig.MASTER_ADDRESS + " to real GridGain node.");

        return createProtocol(addr, conf);
    }

    return null;
}

Example usage for org.apache.hadoop.mapreduce MRConfig MASTER_ADDRESS

Introduction

Prototype

Usage