List of usage examples for org.apache.hadoop.mapreduce.v2 MiniMRYarnCluster MiniMRYarnCluster
public MiniMRYarnCluster(String testName, int noOfNMs)
From source file:com.github.sakserv.minicluster.impl.MRLocalCluster.java
License:Apache License
@Override public void start() throws Exception { LOG.info("MR: Starting MiniMRYarnCluster"); configure();/*from w ww . j a va2 s.co m*/ miniMRYarnCluster = new MiniMRYarnCluster(testName, numNodeManagers); miniMRYarnCluster.serviceInit(configuration); miniMRYarnCluster.init(configuration); miniMRYarnCluster.start(); }
From source file:com.ikanow.aleph2.analytics.hadoop.services.MiniClusterBeJobLauncher.java
License:Apache License
@Override public Configuration getHadoopConfig() { if (_configuration == null) { this._configuration = new Configuration(true); String stagingdir = _configuration.get("yarn.app.mapreduce.am.staging-dir"); logger.debug("staging dir:" + stagingdir); _configuration.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); _configuration.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); try {// ww w . j a v a 2s .c o m mrCluster = new MiniMRYarnCluster("MiniClusterTest", 1); mrCluster.init(_configuration); start(); } catch (Exception e) { logger.error("getConfiguration caused exception", e); } } return _configuration; }
From source file:com.netflix.bdp.s3.TestMRJob.java
License:Apache License
@BeforeClass public static void setupMiniMRCluster() { getConfiguration().set("fs.s3.impl", MockS3FileSystem.class.getName()); S3_OUTPUT_PATH = new Path("s3://bucket-name/output/path"); MR_CLUSTER = new MiniMRYarnCluster("test-s3-multipart-output-committer", 2); MR_CLUSTER.init(getConfiguration()); MR_CLUSTER.start();//from ww w . j a v a2s . co m }
From source file:io.hops.erasure_coding.MrClusterTest.java
License:Apache License
@Override public void setUp() throws Exception { super.setUp(); Configuration conf = new Configuration(getConfig()); mrCluster = new MiniMRYarnCluster(this.getClass().getName(), numDatanode); conf.set("fs.defaultFS", fs.getUri().toString()); YarnAPIStorageFactory.setConfiguration(conf); RMStorageFactory.setConfiguration(conf); mrCluster.init(conf);/*from w w w. ja v a2 s . c o m*/ mrCluster.start(); }
From source file:org.apache.pig.test.TezMiniCluster.java
License:Apache License
@Override public void setupMiniDfsAndMrClusters() { try {/* ww w . j av a 2s . c o m*/ deleteConfFiles(); CONF_DIR.mkdirs(); // Build mini DFS cluster Configuration hdfsConf = new Configuration(false); hdfsConf.addResource("core-default.xml"); hdfsConf.addResource("hdfs-default.xml"); m_dfs = new MiniDFSCluster.Builder(hdfsConf).numDataNodes(2).format(true).racks(null).build(); m_fileSys = m_dfs.getFileSystem(); m_dfs_conf = m_dfs.getConfiguration(0); //Create user home directory m_fileSys.mkdirs(m_fileSys.getWorkingDirectory()); // Write core-site.xml Configuration core_site = new Configuration(false); core_site.set(FileSystem.FS_DEFAULT_NAME_KEY, m_dfs_conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); core_site.writeXml(new FileOutputStream(CORE_CONF_FILE)); Configuration hdfs_site = new Configuration(false); for (Entry<String, String> conf : m_dfs_conf) { if (ArrayUtils.contains(m_dfs_conf.getPropertySources(conf.getKey()), "programatically")) { hdfs_site.set(conf.getKey(), m_dfs_conf.getRaw(conf.getKey())); } } hdfs_site.writeXml(new FileOutputStream(HDFS_CONF_FILE)); // Build mini YARN cluster m_mr = new MiniMRYarnCluster("PigMiniCluster", 2); m_mr.init(m_dfs_conf); m_mr.start(); m_mr_conf = m_mr.getConfig(); m_mr_conf.set(MRConfiguration.FRAMEWORK_NAME, "yarn-tez"); m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, System.getProperty("java.class.path")); m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx2048m"); m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx2048m"); Configuration mapred_site = new Configuration(false); Configuration yarn_site = new Configuration(false); for (Entry<String, String> conf : m_mr_conf) { if (ArrayUtils.contains(m_mr_conf.getPropertySources(conf.getKey()), "programatically")) { if (conf.getKey().contains("yarn")) { yarn_site.set(conf.getKey(), m_mr_conf.getRaw(conf.getKey())); } else if (!conf.getKey().startsWith("dfs")) { mapred_site.set(conf.getKey(), m_mr_conf.getRaw(conf.getKey())); } } } mapred_site.writeXml(new FileOutputStream(MAPRED_CONF_FILE)); yarn_site.writeXml(new FileOutputStream(YARN_CONF_FILE)); // Write tez-site.xml Configuration tez_conf = new Configuration(false); // TODO PIG-3659 - Remove this once memory management is fixed tez_conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, "20"); tez_conf.set("tez.lib.uris", "hdfs:///tez,hdfs:///tez/lib"); // Set to a lower value so that tests don't get stuck for long because of 1 AM running at a time tez_conf.set(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, "20"); // Lower the max task attempts to 2 so that negative tests fail // faster. By default, tasks retry 4 times tez_conf.set(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, "2"); tez_conf.writeXml(new FileOutputStream(TEZ_CONF_FILE)); // Copy tez jars to hdfs m_fileSys.mkdirs(new Path("/tez/lib")); FileFilter fileFilter = new RegexFileFilter("tez-.+\\.jar$"); File[] tezJars = TEZ_LIB_DIR.listFiles(fileFilter); for (int i = 0; i < tezJars.length; i++) { if (tezJars[i].getName().startsWith("tez-api")) { m_fileSys.copyFromLocalFile(new Path(tezJars[i].getAbsoluteFile().toString()), new Path("/tez")); } else { m_fileSys.copyFromLocalFile(new Path(tezJars[i].getAbsoluteFile().toString()), new Path("/tez/lib")); } } m_conf = m_mr_conf; // Turn FetchOptimizer off so that we can actually test Tez m_conf.set(PigConfiguration.OPT_FETCH, System.getProperty("test.opt.fetch", "false")); System.setProperty("junit.hadoop.conf", CONF_DIR.getPath()); System.setProperty("hadoop.log.dir", "build/test/logs"); } catch (IOException e) { throw new RuntimeException(e); } }