List of usage examples for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER
String RM_SCHEDULER
To view the source code for org.apache.hadoop.yarn.conf YarnConfiguration RM_SCHEDULER.
Click Source Link
From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java
License:Apache License
private Configuration createMiniYarnConfig(boolean usePortInName) throws Exception { Configuration conf = new YarnConfiguration(); conf.set("yarn.scheduler.fair.allocation.file", "test-fair-scheduler.xml"); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, FairScheduler.class); //proxy user config String llamaProxyUser = System.getProperty("user.name"); conf.set("hadoop.security.authentication", "simple"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*"); String[] userGroups = new String[] { "g" }; UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, usePortInName); return conf;/*from w w w.jav a2 s.com*/ }
From source file:com.hortonworks.minicluster.MiniHadoopCluster.java
License:Apache License
/** * * @param clusterName/*w ww . j av a 2 s .c o m*/ * @param numNodeManagers */ public MiniHadoopCluster(String clusterName, int numNodeManagers) { super(clusterName); this.testWorkDir = new File("target/MINI_YARN_CLUSTER"); this.resourceManager = new UnsecureResourceManager(); this.numLocalDirs = 1; this.numLogDirs = 1; this.nodeManagers = new NodeManager[numNodeManagers]; this.configuration = new YarnConfiguration(new Configuration()); this.configuration.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); this.configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/MINI_DFS_CLUSTER/data"); try { FileUtils.deleteDirectory(MiniHadoopCluster.this.testWorkDir); } catch (Exception e) { logger.warn("Failed to remove 'target' directory", e); } }
From source file:com.intropro.prairie.unit.yarn.YarnUnit.java
License:Apache License
@Override protected YarnConfiguration gatherConfigs() { YarnConfiguration yarnConfigs = new YarnConfiguration(super.gatherConfigs()); yarnConfigs.set("fs.defaultFS", hdfsUnit.getNamenode()); yarnConfigs.set("mapreduce.task.tmp.dir", getTmpDir().toString()); String user = System.getProperty("user.name"); yarnConfigs.set("hadoop.proxyuser." + user + ".hosts", "*"); yarnConfigs.set("hadoop.proxyuser." + user + ".groups", "*"); yarnConfigs.set("yarn.nodemanager.admin-env", "PATH=$PATH:" + cmdUnit.getPath()); yarnConfigs.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); yarnConfigs.addResource("mapred-site.prairie.xml"); yarnConfigs.addResource("yarn-site.prairie.xml"); return yarnConfigs; }
From source file:com.msd.gin.halyard.common.HBaseServerTestInstance.java
License:Apache License
public static synchronized Configuration getInstanceConfig() throws Exception { if (conf == null) { File zooRoot = File.createTempFile("hbase-zookeeper", ""); zooRoot.delete();/*from ww w . j ava 2 s .com*/ ZooKeeperServer zookeper = new ZooKeeperServer(zooRoot, zooRoot, 2000); ServerCnxnFactory factory = ServerCnxnFactory.createFactory(new InetSocketAddress("localhost", 0), 5000); factory.startup(zookeper); YarnConfiguration yconf = new YarnConfiguration(); String argLine = System.getProperty("argLine"); if (argLine != null) { yconf.set("yarn.app.mapreduce.am.command-opts", argLine.replace("jacoco.exec", "jacocoMR.exec")); } yconf.setBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); yconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MiniMRYarnCluster miniCluster = new MiniMRYarnCluster("testCluster"); miniCluster.init(yconf); yconf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true); miniCluster.start(); File hbaseRoot = File.createTempFile("hbase-root", ""); hbaseRoot.delete(); conf = HBaseConfiguration.create(miniCluster.getConfig()); conf.set(HConstants.HBASE_DIR, hbaseRoot.toURI().toURL().toString()); conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, factory.getLocalPort()); conf.set("hbase.master.hostname", "localhost"); conf.set("hbase.regionserver.hostname", "localhost"); conf.setInt("hbase.master.info.port", -1); conf.set("hbase.fs.tmp.dir", new File(System.getProperty("java.io.tmpdir")).toURI().toURL().toString()); LocalHBaseCluster cluster = new LocalHBaseCluster(conf); cluster.startup(); } return conf; }
From source file:com.splicemachine.test.SpliceTestYarnPlatform.java
License:Apache License
private void configForTesting() throws URISyntaxException { yarnSiteConfigURL = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (yarnSiteConfigURL == null) { throw new RuntimeException("Could not find 'yarn-site.xml' file in classpath"); } else {/* w w w . j a v a 2 s . c om*/ LOG.info("Found 'yarn-site.xml' at " + yarnSiteConfigURL.toURI().toString()); } conf = new YarnConfiguration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.setDouble("yarn.nodemanager.resource.io-spindles", 2.0); conf.set("fs.default.name", "file:///"); conf.set("yarn.nodemanager.container-executor.class", "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"); System.setProperty("zookeeper.sasl.client", "false"); System.setProperty("zookeeper.sasl.serverconfig", "fake"); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, DEFAULT_HEARTBEAT_INTERVAL); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); conf.set("yarn.application.classpath", new File(yarnSiteConfigURL.getPath()).getParent()); }
From source file:edu.uci.ics.asterix.aoya.test.YARNCluster.java
License:Apache License
/** * Instantiates the (Mini) DFS Cluster with the configured number of datanodes. * Post instantiation, data is laoded to HDFS. * Called prior to running the Runtime test suite. *//*from w ww.j a va 2s . c om*/ public void setup() throws Exception { conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml")); conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml")); conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml")); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/integrationts/data"); cleanupLocal(); //this constructor is deprecated in hadoop 2x //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null); miniCluster = new MiniYARNCluster("Asterix_testing", numDataNodes, 1, 1); miniCluster.init(conf); dfs = FileSystem.get(conf); }
From source file:io.hops.metadata.util.DistributedRTClientEvaluation.java
License:Apache License
public DistributedRTClientEvaluation(String rtAddress, int nbSimulatedNM, int hbPeriod, long duration, String output, int startingPort, int nbNMTotal) throws IOException, YarnException, InterruptedException { this.nbNM = nbSimulatedNM; this.hbPeriod = hbPeriod; this.duration = duration; this.output = output; this.nbNMTotal = nbNMTotal; conf = new YarnConfiguration(); conf.setStrings(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, rtAddress); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); //Create NMs//from w w w . j a va 2 s.c o m for (int i = 0; i < nbSimulatedNM; i++) { nmMap.put(i, NodeId.newInstance(InetAddress.getLocalHost().getHostName(), startingPort + i)); } start(); }
From source file:io.hops.metadata.util.DistributedRTRMEvaluation.java
License:Apache License
public DistributedRTRMEvaluation() throws IOException { conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.setBoolean(YarnConfiguration.HOPS_DISTRIBUTED_RT_ENABLED, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YarnAPIStorageFactory.setConfiguration(conf); RMStorageFactory.setConfiguration(conf); }
From source file:io.hops.metadata.util.TestFairSchedulerUtilities.java
License:Apache License
@Before public void setup() throws IOException { try {//www . java2s . co m LOG.info("Setting up Factories"); conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "4000"); YarnAPIStorageFactory.setConfiguration(conf); RMStorageFactory.setConfiguration(conf); RMStorageFactory.getConnector().formatStorage(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, ResourceScheduler.class); // All tests assume only one assignment per node update } catch (StorageInitializtionException ex) { LOG.error(ex); } catch (StorageException ex) { LOG.error(ex); } }
From source file:io.hops.metadata.util.TestHopYarnAPIUtilities.java
License:Apache License
@Before public void setup() throws StorageInitializtionException, StorageException, IOException { LOG.info("Setting up Factories"); conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "4000"); YarnAPIStorageFactory.setConfiguration(conf); RMStorageFactory.setConfiguration(conf); RMUtilities.InitializeDB();//from w w w . j av a 2 s . c om conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); }