List of usage examples for org.apache.hadoop.yarn.server MiniYARNCluster MiniYARNCluster
public MiniYARNCluster(String testName, int numNodeManagers, int numLocalDirs, int numLogDirs)
From source file:co.cask.cdap.operations.yarn.YarnOperationalStatsTest.java
License:Apache License
@Override protected MiniYARNCluster createYarnCluster() throws IOException, InterruptedException, YarnException { MiniYARNCluster yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1); yarnCluster.init(new Configuration()); yarnCluster.start();//w w w . j ava 2 s .com return yarnCluster; }
From source file:com.cloudera.branchreduce.impl.thrift.ClientTest.java
License:Open Source License
@BeforeClass public static void setup() throws InterruptedException, IOException { conf.setInt("yarn.scheduler.fifo.minimum-allocation-mb", 128); conf.set("yarn.nodemanager.vmem-pmem-ratio", "20.0"); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(ClientTest.class.getName(), 1, 1, 1); yarnCluster.init(conf);/* www .j a va 2 s. c om*/ yarnCluster.start(); } try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }
From source file:com.cloudera.kitten.TestKittenDistributedShell.java
License:Open Source License
@BeforeClass public static void setup() throws InterruptedException, IOException { LOG.info("Starting up YARN cluster"); conf.setInt("yarn.scheduler.fifo.minimum-allocation-mb", 128); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestKittenDistributedShell.class.getName(), 1, 1, 1); yarnCluster.init(conf);/*from w w w . ja va2 s. c o m*/ yarnCluster.start(); } try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }
From source file:com.cloudera.llama.am.MiniLlama.java
License:Apache License
private Configuration startMiniHadoop() throws Exception { int clusterNodes = getConf().getInt(MINI_CLUSTER_NODES_KEY, 1); if (System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA) == null) { String testBuildData = new File("target").getAbsolutePath(); System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, testBuildData); }/*from w ww.j a v a 2 s.c om*/ //to trigger hdfs-site.xml registration as default resource new HdfsConfiguration(); Configuration conf = new YarnConfiguration(); String llamaProxyUser = System.getProperty("user.name"); conf.set("hadoop.security.authentication", "simple"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".hosts", "*"); conf.set("hadoop.proxyuser." + llamaProxyUser + ".groups", "*"); String[] userGroups = new String[] { "g" }; UserGroupInformation.createUserForTesting(llamaProxyUser, userGroups); int hdfsPort = 0; String fsUri = conf.get("fs.defaultFS"); if (fsUri != null && !fsUri.equals("file:///")) { int i = fsUri.lastIndexOf(":"); if (i > -1) { try { hdfsPort = Integer.parseInt(fsUri.substring(i + 1)); } catch (Exception ex) { throw new RuntimeException( "Could not parse port from Hadoop's " + "'fs.defaultFS property: " + fsUri); } } } miniHdfs = new MiniDFSCluster(hdfsPort, conf, clusterNodes, !skipDfsFormat, true, null, null); miniHdfs.waitActive(); conf = miniHdfs.getConfiguration(0); miniYarn = new MiniYARNCluster("minillama", clusterNodes, 1, 1); conf.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); miniYarn.init(conf); miniYarn.start(); conf = miniYarn.getConfig(); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); return conf; }
From source file:com.cloudera.llama.am.yarn.TestLlamaAMWithYarn.java
License:Apache License
private void startYarn(Configuration conf, int nodeManagers) throws Exception { miniYarn = new MiniYARNCluster("minillama", nodeManagers, 1, 1); miniYarn.init(conf);/*w ww. j a v a2 s .c om*/ miniYarn.start(); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); Assert.assertTrue("Wait for nodemanagers to connect failed on yarn startup", miniYarn.waitForNodeManagersToConnect(5000)); }
From source file:com.cloudera.llama.nm.TestLlamaNMAuxiliaryService.java
License:Apache License
private void startYarn(Configuration conf) throws Exception { miniYarn = new MiniYARNCluster("llama.nm.plugin", 1, 1, 1); miniYarn.init(conf);/* w w w . j av a2 s. c o m*/ miniYarn.start(); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); }
From source file:com.continuuity.weave.yarn.ClusterTestBase.java
License:Open Source License
protected final void doInit() throws IOException { // Starts Zookeeper zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait();//from w w w. ja va 2 s . c om // Start YARN mini cluster YarnConfiguration config = new YarnConfiguration(new Configuration()); // TODO: Hack config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler" + ".fifo.FifoScheduler"); config.set("yarn.minicluster.fixed.ports", "true"); cluster = new MiniYARNCluster("test-cluster", 1, 1, 1); cluster.init(config); cluster.start(); runnerService = new YarnWeaveRunnerService(config, zkServer.getConnectionStr() + "/weave", new LocalLocationFactory(Files.createTempDir())); runnerService.startAndWait(); }
From source file:com.continuuity.weave.yarn.YarnTestSuite.java
License:Apache License
@BeforeClass public static final void init() throws IOException { // Starts Zookeeper zkServer = InMemoryZKServer.builder().build(); zkServer.startAndWait();/* w w w.j a v a 2 s . co m*/ // Start YARN mini cluster config = new YarnConfiguration(new Configuration()); if (YarnUtils.isHadoop20()) { config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"); } else { config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); config.set("yarn.scheduler.capacity.resource-calculator", "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); } config.set("yarn.minicluster.fixed.ports", "true"); config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1"); config.set("yarn.nodemanager.vmem-check-enabled", "false"); config.set("yarn.scheduler.minimum-allocation-mb", "128"); config.set("yarn.nodemanager.delete.debug-delay-sec", "3600"); cluster = new MiniYARNCluster("test-cluster", 1, 1, 1); cluster.init(config); cluster.start(); runnerService = createWeaveRunnerService(); runnerService.startAndWait(); }
From source file:com.datatorrent.stram.StramMiniClusterTest.java
License:Apache License
@BeforeClass public static void setup() throws InterruptedException, IOException { LOG.info("Starting up YARN cluster"); conf = StramClientUtils.addDTDefaultResources(conf); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); conf.setInt("yarn.nodemanager.vmem-pmem-ratio", 20); // workaround to avoid containers being killed because java allocated too much vmem conf.setStrings("yarn.scheduler.capacity.root.queues", "default"); conf.setStrings("yarn.scheduler.capacity.root.default.capacity", "100"); StringBuilder adminEnv = new StringBuilder(1024); if (System.getenv("JAVA_HOME") == null) { adminEnv.append("JAVA_HOME=").append(System.getProperty("java.home")); adminEnv.append(","); }//from w ww .java 2s . c o m adminEnv.append("MALLOC_ARENA_MAX=4"); // see MAPREDUCE-3068, MAPREDUCE-3065 adminEnv.append(","); adminEnv.append("CLASSPATH=").append(getTestRuntimeClasspath()); conf.set(YarnConfiguration.NM_ADMIN_USER_ENV, adminEnv.toString()); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(StramMiniClusterTest.class.getName(), 1, 1, 1); yarnCluster.init(conf); yarnCluster.start(); } conf = yarnCluster.getConfig(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { LOG.error("Could not find 'yarn-site.xml' dummy file in classpath"); throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } File confFile = new File(url.getPath()); yarnCluster.getConfig().set("yarn.application.classpath", confFile.getParent()); OutputStream os = new FileOutputStream(confFile); LOG.debug("Conf file: {}", confFile); yarnCluster.getConfig().writeXml(os); os.close(); try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }
From source file:com.streamsets.datacollector.MiniSDCTestingUtility.java
License:Apache License
public MiniYARNCluster startMiniYarnCluster(String testName, int numNodeManager, int numLocalDir, int numLogDir, YarnConfiguration yarnConfiguration) { miniYarnCluster = new MiniYARNCluster(testName, numNodeManager, numLocalDir, numLogDir); miniYarnCluster.init(yarnConfiguration); miniYarnCluster.start();//from w w w . j a v a 2s .c o m return miniYarnCluster; }