List of usage examples for org.apache.hadoop.mapred MiniMRCluster MiniMRCluster
public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, int numTaskTrackers, String namenode, int numDir, String[] racks, String[] hosts, UserGroupInformation ugi, JobConf conf) throws IOException
From source file:com.alexholmes.hadooputils.test.MiniHadoop.java
License:Apache License
/** * Creates a {@link MiniMRCluster} and {@link MiniDFSCluster} all working within * the directory supplied in {@code tmpDir}. * * The DFS will be formatted regardless if there was one or not before in the * given location./*from w w w .ja v a2 s. co m*/ * * @param config the Hadoop configuration * @param taskTrackers number of task trackers to start * @param dataNodes number of data nodes to start * @param tmpDir the temporary directory which the Hadoop cluster will use for storage * @throws IOException thrown if the base directory cannot be set. */ public MiniHadoop(final Configuration config, final int taskTrackers, final int dataNodes, final File tmpDir) throws IOException { if (taskTrackers < 1) { throw new IllegalArgumentException("Invalid taskTrackers value, must be greater than 0"); } if (dataNodes < 1) { throw new IllegalArgumentException("Invalid dataNodes value, must be greater than 0"); } config.set("hadoop.tmp.dir", tmpDir.getAbsolutePath()); if (tmpDir.exists()) { FileUtils.forceDelete(tmpDir); } FileUtils.forceMkdir(tmpDir); // used by MiniDFSCluster for DFS storage System.setProperty("test.build.data", new File(tmpDir, "data").getAbsolutePath()); // required by JobHistory.initLogDir System.setProperty("hadoop.log.dir", new File(tmpDir, "logs").getAbsolutePath()); JobConf jobConfig = new JobConf(config); dfsCluster = new MiniDFSCluster(jobConfig, dataNodes, true, null); fileSystem = dfsCluster.getFileSystem(); mrCluster = new MiniMRCluster(0, 0, taskTrackers, fileSystem.getUri().toString(), 1, null, null, null, jobConfig); }
From source file:org.apache.hcatalog.hbase.ManyMiniCluster.java
License:Apache License
private void setupMRCluster() { try {/* w ww. j a v a 2 s. c o m*/ final int jobTrackerPort = findFreePort(); final int taskTrackerPort = findFreePort(); if (jobConf == null) jobConf = new JobConf(); jobConf.setInt("mapred.submit.replication", 1); jobConf.set("yarn.scheduler.capacity.root.queues", "default"); jobConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); //conf.set("hadoop.job.history.location",new File(workDir).getAbsolutePath()+"/history"); System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); mrCluster = new MiniMRCluster(jobTrackerPort, taskTrackerPort, numTaskTrackers, getFileSystem().getUri().toString(), numTaskTrackers, null, null, null, jobConf); jobConf = mrCluster.createJobConf(); } catch (IOException e) { throw new IllegalStateException("Failed to Setup MR Cluster", e); } }
From source file:org.apache.oozie.test.XTestCase.java
License:Apache License
private void setUpEmbeddedHadoop(String testCaseDir) throws Exception { if (dfsCluster == null && mrCluster == null) { if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", testCaseDir); }// w ww. jav a 2s .c o m int taskTrackers = 2; int dataNodes = 2; String oozieUser = getOozieUser(); JobConf conf = createDFSConfig(); String[] userGroups = new String[] { getTestGroup(), getTestGroup2() }; UserGroupInformation.createUserForTesting(oozieUser, userGroups); UserGroupInformation.createUserForTesting(getTestUser(), userGroups); UserGroupInformation.createUserForTesting(getTestUser2(), userGroups); UserGroupInformation.createUserForTesting(getTestUser3(), new String[] { "users" }); try { dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("target/test-data")); fileSystem.mkdirs(new Path("target/test-data" + "/minicluster/mapred")); fileSystem.mkdirs(new Path("/user")); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/hadoop/mapred/system")); fileSystem.setPermission(new Path("target/test-data"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("target/test-data" + "/minicluster"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("target/test-data" + "/minicluster/mapred"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/hadoop/mapred/system"), FsPermission.valueOf("-rwx------")); String nnURI = fileSystem.getUri().toString(); int numDirs = 1; String[] racks = null; String[] hosts = null; mrCluster = new MiniMRCluster(0, 0, taskTrackers, nnURI, numDirs, racks, hosts, null, conf); JobConf jobConf = mrCluster.createJobConf(); System.setProperty(OOZIE_TEST_JOB_TRACKER, jobConf.get("mapred.job.tracker")); String rmAddress = jobConf.get("yarn.resourcemanager.address"); if (rmAddress != null) { System.setProperty(OOZIE_TEST_JOB_TRACKER, rmAddress); } System.setProperty(OOZIE_TEST_NAME_NODE, jobConf.get("fs.default.name")); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); } catch (Exception ex) { shutdownMiniCluster(); throw ex; } new MiniClusterShutdownMonitor().start(); } }
From source file:org.apache.solr.hadoop.MorphlineBasicMiniMRTest.java
License:Apache License
@BeforeClass public static void setupClass() throws Exception { if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", "target"); }/*from w w w. j a va 2 s . c o m*/ int taskTrackers = 2; int dataNodes = 2; // String proxyUser = System.getProperty("user.name"); // String proxyGroup = "g"; // StringBuilder sb = new StringBuilder(); // sb.append("127.0.0.1,localhost"); // for (InetAddress i : InetAddress.getAllByName(InetAddress.getLocalHost().getHostName())) { // sb.append(",").append(i.getCanonicalHostName()); // } System.setProperty("solr.hdfs.blockcache.enabled", "false"); JobConf conf = new JobConf(); conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/user")); fileSystem.mkdirs(new Path("/hadoop/mapred/system")); fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/hadoop/mapred/system"), FsPermission.valueOf("-rwx------")); String nnURI = fileSystem.getUri().toString(); int numDirs = 1; String[] racks = null; String[] hosts = null; mrCluster = new MiniMRCluster(0, 0, taskTrackers, nnURI, numDirs, racks, hosts, null, conf); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); }
From source file:org.apache.solr.hadoop.MorphlineGoLiveMiniMRTest.java
License:Apache License
@BeforeClass public static void setupClass() throws Exception { // if (isYarn()) { // org.junit.Assume.assumeTrue(false); // ignore test on Yarn until CDH-10420 is fixed // }/*from w ww.ja v a 2s . c o m*/ if (System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", "target"); } int taskTrackers = 2; int dataNodes = 2; System.setProperty("solr.hdfs.blockcache.enabled", "false"); JobConf conf = new JobConf(); conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); createTempDir(); System.setProperty("test.build.data", dataDir + File.separator + "hdfs" + File.separator + "build"); System.setProperty("test.cache.data", dataDir + File.separator + "hdfs" + File.separator + "cache"); dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/user")); fileSystem.mkdirs(new Path("/hadoop/mapred/system")); fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx")); fileSystem.setPermission(new Path("/hadoop/mapred/system"), FsPermission.valueOf("-rwx------")); String nnURI = fileSystem.getUri().toString(); int numDirs = 1; String[] racks = null; String[] hosts = null; mrCluster = new MiniMRCluster(0, 0, taskTrackers, nnURI, numDirs, racks, hosts, null, conf); ProxyUsers.refreshSuperUserGroupsConfiguration(conf); }
From source file:org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner.java
License:Apache License
@SuppressWarnings("deprecation") @Override/*w ww . jav a 2s . co m*/ public void start() throws Exception { System.setProperty("test.build.data", getDataDir()); LOG.info("test.build.data set to: " + getDataDir()); System.setProperty("hadoop.log.dir", getLogDir()); LOG.info("log dir set to: " + getLogDir()); // Start DFS server LOG.info("Starting DFS cluster..."); dfsCluster = new MiniDFSCluster(config, 1, true, null); if (dfsCluster.isClusterUp()) { LOG.info("Started DFS cluster on port: " + dfsCluster.getNameNodePort()); } else { LOG.error("Could not start DFS cluster"); } // Start MR server LOG.info("Starting MR cluster"); mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, new JobConf(config)); LOG.info("Started MR cluster"); config = prepareConfiguration(mrCluster.createJobConf()); }
From source file:org.icgc.dcc.submission.MiniHadoop.java
License:Open Source License
/** * Creates a {@link MiniMRCluster} and {@link MiniDFSCluster} all working within the directory supplied in * {@code tmpDir}./*from w w w . j a v a2s . com*/ * * The DFS will be formatted regardless if there was one or not before in the given location. * * @param config the Hadoop configuration * @param taskTrackers number of task trackers to start * @param dataNodes number of data nodes to start * @param tmpDir the temporary directory which the Hadoop cluster will use for storage * @throws IOException thrown if the base directory cannot be set. */ public MiniHadoop(final Configuration config, final int taskTrackers, final int dataNodes, final File tmpDir) throws IOException { config.set("hadoop.tmp.dir", tmpDir.getAbsolutePath()); if (tmpDir.exists()) { FileUtils.forceDelete(tmpDir); } FileUtils.forceMkdir(tmpDir); // used by MiniDFSCluster for DFS storage System.setProperty("test.build.data", new File(tmpDir, "data").getAbsolutePath()); // required by JobHistory.initLogDir System.setProperty("hadoop.log.dir", new File(tmpDir, "logs").getAbsolutePath()); // Configure logging // TIP: Use `find /tmp/hadoop -L -name stdout | xargs cat | less -S` to see all the logs ((Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME)).setLevel(Level.INFO); ((Logger) LoggerFactory.getLogger("org.apache.hadoop")).setLevel(Level.WARN); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.conf")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.ipc")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.hdfs")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.mapred.DefaultTaskController")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.util.ProcessTree")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.util.ProcessTree")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.util.NativeCodeLoader")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.metrics2.impl.MetricsConfig")).setLevel(Level.ERROR); ((Logger) LoggerFactory.getLogger("org.apache.hadoop.mapred.TaskTracker")).setLevel(Level.OFF); ((Logger) LoggerFactory.getLogger("org.mortbay")).setLevel(Level.WARN); ((Logger) LoggerFactory.getLogger("mapreduce")).setLevel(Level.ERROR); JobConf jobConfig = new JobConf(config); dfsCluster = new MiniDFSCluster.Builder(jobConfig).numDataNodes(dataNodes).format(true).build(); fileSystem = dfsCluster.getFileSystem(); mrCluster = new MiniMRCluster(0, 0, taskTrackers, fileSystem.getUri().toString(), 1, null, null, null, jobConfig); }
From source file:org.lilyproject.hadooptestfw.fork.HBaseTestingUtility.java
License:Apache License
/** * Starts a <code>MiniMRCluster</code>. * * @param servers The number of <code>TaskTracker</code>'s to start. * @throws IOException When starting the cluster fails. *///from www . j a v a2s .c om public void startMiniMapReduceCluster(final int servers) throws IOException { LOG.info("Starting mini mapreduce cluster..."); // These are needed for the new and improved Map/Reduce framework Configuration c = getConfiguration(); String logDir = c.get("hadoop.log.dir"); String tmpDir = c.get("hadoop.tmp.dir"); if (logDir == null) { logDir = tmpDir; } System.setProperty("hadoop.log.dir", logDir); c.set("mapred.output.dir", tmpDir); // Lily change: fix port number + set host name (+ fix # of servers to 1) + use custom jobconf so that // hadoop dir properties get passed on. JobConf jobConf = new JobConf(conf); mrCluster = new MiniMRCluster(8021, 0, 1, FileSystem.get(conf).getUri().toString(), 1, null, new String[] { "localhost" }, null, jobConf); // End Lily change LOG.info("Mini mapreduce cluster started"); JobConf mrClusterJobConf = mrCluster.createJobConf(); // In hadoop2, YARN/MR2 starts a mini cluster with its own conf instance and updates settings. // Our HBase MR jobs need several of these settings in order to properly run. So we copy the // necessary config properties here. YARN-129 required adding a few properties. c.set("mapred.job.tracker", mrClusterJobConf.get("mapred.job.tracker")); /* this for mrv2 support */ conf.set("mapreduce.framework.name", "yarn"); conf.setBoolean("yarn.is.minicluster", true); String rmAdress = mrClusterJobConf.get("yarn.resourcemanager.address"); if (rmAdress != null) { conf.set("yarn.resourcemanager.address", rmAdress); } String schedulerAdress = mrClusterJobConf.get("yarn.resourcemanager.scheduler.address"); if (schedulerAdress != null) { conf.set("yarn.resourcemanager.scheduler.address", schedulerAdress); } }