List of usage examples for org.apache.hadoop.yarn.server MiniYARNCluster getHostname
public static String getHostname()
From source file:co.cask.cdap.operations.yarn.YarnRMHAOperationalStatsTest.java
License:Apache License
@Override protected MiniYARNCluster createYarnCluster() throws IOException, InterruptedException, YarnException { Configuration hConf = new Configuration(); hConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); String hostname = MiniYARNCluster.getHostname(); for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { hConf.set(HAUtil.addSuffix(confKey, "rm0"), hostname + ":" + Networks.getRandomPort()); hConf.set(HAUtil.addSuffix(confKey, "rm1"), hostname + ":" + Networks.getRandomPort()); }//from w ww .ja va 2s. c om MiniYARNCluster yarnCluster = new MiniYARNCluster(getClass().getName(), 2, 2, 2, 2); yarnCluster.init(hConf); yarnCluster.start(); yarnCluster.getResourceManager(0).getRMContext().getRMAdminService().transitionToActive( new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource.REQUEST_BY_USER)); return yarnCluster; }
From source file:com.google.mr4c.hadoop.yarn.YarnTestBinding.java
License:Open Source License
private void startMrCluster() throws IOException { Configuration conf = new JobConf(); FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri()); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); String addr = MiniYARNCluster.getHostname() + ":0"; conf.set(YarnConfiguration.RM_ADDRESS, addr); conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr); m_mrCluster = MiniMRClientClusterFactory.create(HadoopTestUtils.class, "MR4CTests", 1, // num node managers conf);/*from w w w .jav a 2s .com*/ // make sure startup is finished for (int i = 0; i < 60; i++) { String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); if (newAddr.equals(addr)) { s_log.warn("MiniYARNCluster startup not complete"); try { Thread.sleep(1000); } catch (InterruptedException ie) { throw new IOException(ie); } } else { s_log.info("MiniYARNCluster now available at {}", newAddr); return; } } throw new IOException("MiniYARNCluster taking too long to startup"); }
From source file:edu.uci.ics.asterix.aoya.test.AsterixYARNInstanceUtil.java
License:Apache License
public YarnConfiguration setUp() throws Exception { File asterixProjectDir = new File(System.getProperty("user.dir")); File installerTargetDir = new File(asterixProjectDir, "target"); String[] dirsInTarget = installerTargetDir.list(new FilenameFilter() { @Override//from w w w . j a va 2 s .c o m public boolean accept(File dir, String name) { return new File(dir, name).isDirectory() && name.startsWith("asterix-yarn") && name.endsWith("binary-assembly"); } }); if (dirsInTarget.length != 1) { throw new IllegalStateException("Could not find binary to run YARN integration test with"); } aoyaHome = installerTargetDir.getAbsolutePath() + File.separator + dirsInTarget[0]; File asterixServerInstallerDir = new File(aoyaHome, "asterix"); String[] zipsInFolder = asterixServerInstallerDir.list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.startsWith("asterix-server") && name.endsWith("binary-assembly.zip"); } }); if (zipsInFolder.length != 1) { throw new IllegalStateException("Could not find server binary to run YARN integration test with"); } aoyaServerPath = asterixServerInstallerDir.getAbsolutePath() + File.separator + zipsInFolder[0]; configPath = aoyaHome + File.separator + "configs" + File.separator + "local.xml"; parameterPath = aoyaHome + File.separator + "conf" + File.separator + "base-asterix-configuration.xml"; YARNCluster.getInstance().setup(); appConf = new YarnConfiguration(); File baseDir = new File("./target/hdfs/").getAbsoluteFile(); FileUtil.fullyDelete(baseDir); appConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(appConf); MiniDFSCluster hdfsCluster = builder.build(); miniCluster = YARNCluster.getInstance().getCluster(); appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort()); miniCluster.init(appConf); Cluster defaultConfig = Utils.parseYarnClusterConfig(configPath); for (Node n : defaultConfig.getNode()) { n.setClusterIp(MiniYARNCluster.getHostname()); } defaultConfig.getMasterNode().setClusterIp(MiniYARNCluster.getHostname()); configPath = "target" + File.separator + "localized-aoya-config.xml"; Utils.writeYarnClusterConfig(configPath, defaultConfig); miniCluster.start(); appConf = new YarnConfiguration(miniCluster.getConfig()); appConf.set("fs.defaultFS", "hdfs://localhost:" + hdfsCluster.getNameNodePort()); //TODO:why must I do this!? what is not being passed properly via environment variables??? appConf.writeXml(new FileOutputStream("target" + File.separator + "yarn-site.xml")); //once the cluster is created, you can get its configuration //with the binding details to the cluster added from the minicluster FileSystem fs = FileSystem.get(appConf); Path instanceState = new Path(fs.getHomeDirectory(), AsterixYARNClient.CONF_DIR_REL + INSTANCE_NAME + "/"); fs.delete(instanceState, true); Assert.assertFalse(fs.exists(instanceState)); File outdir = new File(PATH_ACTUAL); outdir.mkdirs(); return appConf; }
From source file:io.hops.tensorflow.TestCluster.java
License:Apache License
protected void setupInternal(int numNodeManager) throws Exception { LOG.info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.set("yarn.log.dir", "target"); conf.set("yarn.log-aggregation-enable", "true"); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName()); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_GPU_RESOURCE_ENABLED, false); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestCluster.class.getSimpleName(), 1, numNodeManager, 1, 1); yarnCluster.init(conf);/* w ww .ja v a2 s.c o m*/ yarnCluster.start(); conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort()); waitForNMsToRegister(); URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"); } Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(url.getPath()).getParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(url.getPath())); os.write(bytesOut.toByteArray()); os.close(); } FileContext fsContext = FileContext.getLocalFSFileContext(); fsContext.delete(new Path(conf.get("yarn.timeline-service.leveldb-timeline-store.path")), true); try { Thread.sleep(2000); } catch (InterruptedException e) { LOG.info("setup thread sleep interrupted. message=" + e.getMessage()); } }