List of usage examples for org.apache.hadoop.mapred MiniMRClientClusterFactory create
public static MiniMRClientCluster create(Class<?> caller, int noOfNMs, Configuration conf) throws IOException
From source file:cascading.platform.hadoop2.Hadoop2MR1Platform.java
License:Open Source License
@Override public synchronized void setUp() throws IOException { if (configuration != null) return;/*w w w. j av a 2 s . c o m*/ if (!isUseCluster()) { LOG.info("not using cluster"); configuration = new JobConf(); // enforce settings to make local mode behave the same across distributions configuration.set("fs.defaultFS", "file:///"); configuration.set("mapreduce.framework.name", "local"); configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/" + "build/tmp/cascading/staging"); String stagingDir = configuration.get("mapreduce.jobtracker.staging.root.dir"); if (Util.isEmpty(stagingDir)) configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/build/tmp/cascading/staging"); fileSys = FileSystem.get(configuration); } else { LOG.info("using cluster"); if (Util.isEmpty(System.getProperty("hadoop.log.dir"))) System.setProperty("hadoop.log.dir", "build/test/log"); if (Util.isEmpty(System.getProperty("hadoop.tmp.dir"))) System.setProperty("hadoop.tmp.dir", "build/test/tmp"); new File(System.getProperty("hadoop.log.dir")).mkdirs(); // ignored JobConf conf = new JobConf(); if (!Util.isEmpty(System.getProperty("mapred.jar"))) { LOG.info("using a remote cluster with jar: {}", System.getProperty("mapred.jar")); configuration = conf; ((JobConf) configuration).setJar(System.getProperty("mapred.jar")); if (!Util.isEmpty(System.getProperty("fs.default.name"))) { LOG.info("using {}={}", "fs.default.name", System.getProperty("fs.default.name")); configuration.set("fs.default.name", System.getProperty("fs.default.name")); } if (!Util.isEmpty(System.getProperty("mapred.job.tracker"))) { LOG.info("using {}={}", "mapred.job.tracker", System.getProperty("mapred.job.tracker")); configuration.set("mapred.job.tracker", System.getProperty("mapred.job.tracker")); } if (!Util.isEmpty(System.getProperty("fs.defaultFS"))) { LOG.info("using {}={}", "fs.defaultFS", System.getProperty("fs.defaultFS")); configuration.set("fs.defaultFS", System.getProperty("fs.defaultFS")); } if (!Util.isEmpty(System.getProperty("yarn.resourcemanager.address"))) { LOG.info("using {}={}", "yarn.resourcemanager.address", System.getProperty("yarn.resourcemanager.address")); configuration.set("yarn.resourcemanager.address", System.getProperty("yarn.resourcemanager.address")); } if (!Util.isEmpty(System.getProperty("mapreduce.jobhistory.address"))) { LOG.info("using {}={}", "mapreduce.jobhistory.address", System.getProperty("mapreduce.jobhistory.address")); configuration.set("mapreduce.jobhistory.address", System.getProperty("mapreduce.jobhistory.address")); } configuration.set("mapreduce.user.classpath.first", "true"); // use test dependencies configuration.set("mapreduce.framework.name", "yarn"); fileSys = FileSystem.get(configuration); } else { conf.setBoolean("yarn.is.minicluster", true); // conf.setInt( "yarn.nodemanager.delete.debug-delay-sec", -1 ); // conf.set( "yarn.scheduler.capacity.root.queues", "default" ); // conf.set( "yarn.scheduler.capacity.root.default.capacity", "100" ); // disable blacklisting hosts not to fail localhost during unit tests conf.setBoolean("yarn.app.mapreduce.am.job.node-blacklisting.enable", false); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); FileSystem.setDefaultUri(conf, fileSys.getUri()); mr = MiniMRClientClusterFactory.create(this.getClass(), 4, conf); configuration = mr.getConfig(); } configuration.set("mapred.child.java.opts", "-Xmx512m"); configuration.setInt("mapreduce.job.jvm.numtasks", -1); configuration.setInt("mapreduce.client.completion.pollinterval", 50); configuration.setInt("mapreduce.client.progressmonitor.pollinterval", 50); configuration.setBoolean("mapreduce.map.speculative", false); configuration.setBoolean("mapreduce.reduce.speculative", false); } configuration.setInt("mapreduce.job.maps", numMappers); configuration.setInt("mapreduce.job.reduces", numReducers); Map<Object, Object> globalProperties = getGlobalProperties(); if (logger != null) globalProperties.put("log4j.logger", logger); FlowProps.setJobPollingInterval(globalProperties, 10); // should speed up tests Hadoop2MR1Planner.copyProperties(configuration, globalProperties); // copy any external properties Hadoop2MR1Planner.copyConfiguration(properties, configuration); // put all properties on the jobconf }
From source file:com.accumulobook.advanced.mapreduce.MiniMRClusterRunner.java
License:Apache License
public void setup() throws IOException { Configuration conf = new YarnConfiguration(); cluster = MiniMRClientClusterFactory.create(this.getClass(), 2, conf); }
From source file:com.linkedin.hadoop.example.minicluster.MapReduceClusterTestBase.java
License:Apache License
/** * Initializes the mini-cluster to use with unit tests. * * @throws Exception If there is a problem initializing the mini-cluster *//* w w w .j av a2 s .c o m*/ public void beforeClass() throws Exception { // Make sure the log folder exists or your tests will fail new File("test-logs").mkdirs(); System.setProperty("hadoop.log.dir", "test-logs"); // Setup a new Configuration for the cluster Configuration conf = new Configuration(); conf.set("mapreduce.framework.name", "local"); _logger.info("*** Starting Mini DFS Cluster"); _dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(_dataNodes).build(); _fileSystem = _dfsCluster.getFileSystem(); _logger.info("*** Starting Mini MR Cluster"); _mrCluster = MiniMRClientClusterFactory.create(this.getClass(), _dataNodes, conf); }
From source file:com.mycompany.app.TestStagingDirectoryPermissions.java
License:Apache License
@Test public void perms() throws IOException, InterruptedException { MiniDFSCluster minidfs = null;//from w w w . j a va2s .com FileSystem fs = null; MiniMRClientCluster minimr = null; try { Configuration conf = new Configuration(true); conf.set("fs.permission.umask-mode", "0077"); minidfs = new MiniDFSCluster.Builder(conf).build(); minidfs.waitActive(); fs = minidfs.getFileSystem(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, fs.getUri().toString()); Path p = path("/in"); fs.mkdirs(p); FSDataOutputStream os = fs.create(new Path(p, "input.txt")); os.write("hello!".getBytes("UTF-8")); os.close(); String user = UserGroupInformation.getCurrentUser().getUserName(); Path home = new Path("/User/" + user); fs.mkdirs(home); minimr = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); JobConf job = new JobConf(minimr.getConfig()); job.setJobName("PermsTest"); JobClient client = new JobClient(job); FileInputFormat.addInputPath(job, p); FileOutputFormat.setOutputPath(job, path("/out")); job.setInputFormat(TextInputFormat.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setMapperClass(MySleepMapper.class); job.setNumReduceTasks(1); RunningJob submittedJob = client.submitJob(job); // Sleep for a bit to let localization finish System.out.println("Sleeping..."); Thread.sleep(3 * 1000l); System.out.println("Done sleeping..."); assertFalse(UserGroupInformation.isSecurityEnabled()); Path stagingRoot = path("/tmp/hadoop-yarn/staging/" + user + "/.staging/"); assertTrue(fs.exists(stagingRoot)); assertEquals(1, fs.listStatus(stagingRoot).length); Path staging = fs.listStatus(stagingRoot)[0].getPath(); Path jobXml = path(staging + "/job.xml"); assertTrue(fs.exists(jobXml)); FileStatus fileStatus = fs.getFileStatus(jobXml); System.out.println("job.xml permission = " + fileStatus.getPermission()); assertTrue(fileStatus.getPermission().getOtherAction().implies(FsAction.READ)); assertTrue(fileStatus.getPermission().getGroupAction().implies(FsAction.READ)); submittedJob.waitForCompletion(); } finally { if (minimr != null) { minimr.stop(); } if (fs != null) { fs.close(); } if (minidfs != null) { minidfs.shutdown(true); } } }
From source file:etl.cmd.test.XTestCase.java
License:Apache License
private void setUpEmbeddedHadoop(String testCaseDir) throws Exception { if (dfsCluster == null && mrCluster == null) { if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", testCaseDir); }/*from w w w . j a v a 2 s.c om*/ String oozieUser = getOozieUser(); JobConf conf = createDFSConfig(); String[] userGroups = new String[] { getTestGroup(), getTestGroup2() }; UserGroupInformation.createUserForTesting(oozieUser, userGroups); UserGroupInformation.createUserForTesting(getTestUser(), userGroups); UserGroupInformation.createUserForTesting(getTestUser2(), userGroups); UserGroupInformation.createUserForTesting(getTestUser3(), new String[] { "users" }); try { MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); dfsCluster = builder.build(); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("target/test-data")); fileSystem.mkdirs(new Path("target/test-data" + "/minicluster/mapred")); fileSystem.mkdirs(new Path("/user")); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/hadoop/mapred/system")); fileSystem.setPermission(new Path("target/test-data"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("target/test-data" + "/minicluster"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("target/test-data" + "/minicluster/mapred"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/hadoop/mapred/system"), FsPermission.valueOf("-rwx------")); mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); Configuration jobConf = mrCluster.getConfig(); System.setProperty(OOZIE_TEST_JOB_TRACKER, jobConf.get("mapreduce.jobtracker.address")); String rmAddress = jobConf.get("yarn.resourcemanager.address"); log.info("Job tracker: " + rmAddress); if (rmAddress != null) { System.setProperty(OOZIE_TEST_JOB_TRACKER, rmAddress); } System.setProperty(OOZIE_TEST_NAME_NODE, jobConf.get("fs.defaultFS")); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); } catch (Exception ex) { shutdownMiniCluster(); throw ex; } new MiniClusterShutdownMonitor().start(); } }
From source file:io.aos.mapreduce.count.WordCountToolTest.java
License:Apache License
@BeforeClass public static void setup() throws IOException { Properties props = new Properties(); InputStream is = ClassLoader.getSystemResourceAsStream("hdfs-conf.properties"); props.load(is);// ww w . j a v a2 s . com for (Entry<Object, Object> entry : props.entrySet()) { System.setProperty((String) entry.getKey(), (String) entry.getValue()); } Map<String, String> envMap = new HashMap<String, String>(); envMap.put("JAVA_HOME", System.getProperty("java.home")); setEnv(envMap); final Configuration conf = new Configuration(); final Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "/tmp")); testdir = new Path(TEST_ROOT_DIR, "TestMiniMRClientCluster"); inDir = new Path(testdir, "in"); outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.getLocal(conf); if (fs.exists(testdir) && !fs.delete(testdir, true)) { throw new IOException("Could not delete " + testdir); } if (!fs.mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir); } for (int i = 0; i < inFiles.length; i++) { inFiles[i] = new Path(inDir, "part_" + i); createFile(inFiles[i], conf); } // create the mini cluster to be used for the tests mrCluster = MiniMRClientClusterFactory.create(WordCountToolTest.class, 1, new Configuration()); }
From source file:io.aos.mapreduce.grep.GrepToolTest.java
License:Apache License
@BeforeClass public static void setup() throws IOException { Properties props = new Properties(); InputStream is = ClassLoader.getSystemResourceAsStream("hdfs-conf.properties"); props.load(is);//from w w w .j av a2 s . c o m for (Entry<Object, Object> entry : props.entrySet()) { System.setProperty((String) entry.getKey(), (String) entry.getValue()); } Map<String, String> envMap = new HashMap<String, String>(); envMap.put("JAVA_HOME", System.getProperty("java.home")); setEnv(envMap); final Configuration conf = new Configuration(); final Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "/tmp")); testdir = new Path(TEST_ROOT_DIR, "TestMiniMRClientCluster"); inDir = new Path(testdir, "in"); outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.getLocal(conf); if (fs.exists(testdir) && !fs.delete(testdir, true)) { throw new IOException("Could not delete " + testdir); } if (!fs.mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir); } for (int i = 0; i < inFiles.length; i++) { inFiles[i] = new Path(inDir, "part_" + i); createFile(inFiles[i], conf); } // create the mini cluster to be used for the tests mrCluster = MiniMRClientClusterFactory.create(GrepToolTest.class, 1, new Configuration()); }