List of usage examples for org.apache.hadoop.hdfs MiniDFSCluster MiniDFSCluster
@Deprecated public MiniDFSCluster(Configuration conf, int numDataNodes, boolean format, String[] racks) throws IOException
From source file:TestFuseDFS.java
License:Apache License
static public void startStuff() { try {/* ww w . j av a 2s . co m*/ Configuration conf = new Configuration(); conf.setBoolean("dfs.permissions", false); cluster = new MiniDFSCluster(conf, 1, true, null); fileSys = (DistributedFileSystem) cluster.getFileSystem(); assertTrue(fileSys.getFileStatus(new Path("/")).isDir()); mount(mpoint, fileSys.getUri()); } catch (Exception e) { e.printStackTrace(); } }
From source file:PerformanceEvaluation.java
License:Apache License
private void runTest(final Class<? extends Test> cmd) throws IOException, InterruptedException, ClassNotFoundException { MiniHBaseCluster hbaseMiniCluster = null; MiniDFSCluster dfsCluster = null;/*from w w w .j a v a2 s . c o m*/ MiniZooKeeperCluster zooKeeperCluster = null; if (this.miniCluster) { dfsCluster = new MiniDFSCluster(conf, 2, true, (String[]) null); zooKeeperCluster = new MiniZooKeeperCluster(); int zooKeeperPort = zooKeeperCluster.startup(new File(System.getProperty("java.io.tmpdir"))); // mangle the conf so that the fs parameter points to the minidfs we // just started up FileSystem fs = dfsCluster.getFileSystem(); conf.set("fs.default.name", fs.getUri().toString()); conf.set("hbase.zookeeper.property.clientPort", Integer.toString(zooKeeperPort)); Path parentdir = fs.getHomeDirectory(); conf.set(HConstants.HBASE_DIR, parentdir.toString()); fs.mkdirs(parentdir); FSUtils.setVersion(fs, parentdir); hbaseMiniCluster = new MiniHBaseCluster(this.conf, N); } try { if (N == 1) { // If there is only one client and one HRegionServer, we assume nothing // has been set up at all. runNIsOne(cmd); } else { // Else, run runNIsMoreThanOne(cmd); } } finally { if (this.miniCluster) { if (hbaseMiniCluster != null) hbaseMiniCluster.shutdown(); if (zooKeeperCluster != null) zooKeeperCluster.shutdown(); HBaseTestCase.shutdownDfs(dfsCluster); } } }
From source file:JaqlShell.java
License:Apache License
/** * @param dir//from www . j ava 2s . c o m * @param numNodes * @param format * @throws Exception */ public void init(String dir, int numNodes) throws Exception { String vInfo = VersionInfo.getVersion(); System.setProperty("test.build.data", dir); m_conf = new Configuration(); // setup conf according to the Hadoop version if (vInfo.indexOf("0.20") < 0) { throw new Exception("Unsupported Hadoop version: " + vInfo); } // setup the mini dfs cluster m_fs = new MiniDFSCluster(m_conf, numNodes, true, (String[]) null); FileSystem filesystem = m_fs.getFileSystem(); m_conf.set("fs.default.name", filesystem.getUri().toString()); Path parentdir = filesystem.getHomeDirectory(); filesystem.mkdirs(parentdir); //FSUtils.setVersion(filesystem, parentdir); // setup hbase cluster (only if OS is not windows) // if(!System.getProperty("os.name").toLowerCase().contains("win")) { // m_conf.set(HConstants.HBASE_DIR, parentdir.toString()); // Path hdfsTestDir = filesystem.makeQualified(new Path(m_conf.get(HConstants.HBASE_DIR))); // // // prime the hdfs for hbase information... // HRegion root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, hdfsTestDir, (HBaseConfiguration)m_conf); // HRegion meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, hdfsTestDir, (HBaseConfiguration)m_conf); // HRegion.addRegionToMETA(root, meta); // // // ... and close the root and meta // if (meta != null) { // meta.close(); // meta.getLog().closeAndDelete(); // } // if (root != null) { // root.close(); // root.getLog().closeAndDelete(); // } // // try // { // this.zooKeeperCluster = new MiniZooKeeperCluster(); // File testDir = new File(dir); // int clientPort = this.zooKeeperCluster.startup(testDir); // m_conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort)); // } catch(Exception e) { // LOG.error("Unable to startup zookeeper"); // throw new IOException(e); // } // try { // // start the mini cluster // m_base = new MiniHBaseCluster((HBaseConfiguration)m_conf, numNodes); // } catch(Exception e) { // LOG.error("Unable to startup hbase"); // throw new IOException(e); // } // try { // // opening the META table ensures that cluster is running // new HTable((HBaseConfiguration)m_conf, HConstants.META_TABLE_NAME); // // //setupOverride(conf); // } // catch (Exception e) // { // LOG.warn("Could not verify that hbase is up", e); // } // setupOverride(); // } m_mr = startMRCluster(numNodes, m_fs.getFileSystem().getName(), m_conf); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); // make the home directory if it does not exist Path hd = fs.getWorkingDirectory(); if (!fs.exists(hd)) fs.mkdirs(hd); // make the $USER/_temporary directory if it does not exist Path tmpPath = new Path(hd, "_temporary"); if (!fs.exists(tmpPath)) fs.mkdirs(tmpPath); // if (m_base != null) // { // try { // m_admin = new HBaseAdmin((HBaseConfiguration) m_conf); // HTableDescriptor[] tables = m_admin.listTables(); // if (tables != null) // { // for (int i = 0; i < tables.length; i++) // { // m_admin.enableTable(tables[i].getName()); // } // } // } catch(Exception e) { // LOG.warn("failed to enable hbase tables"); // } // } }
From source file:cascading.ClusterTestCase.java
License:Open Source License
public void setUp() throws IOException { if (jobConf != null) return;//from w w w .jav a 2 s .c o m if (!enableCluster) { jobConf = new JobConf(); } else { System.setProperty("test.build.data", "build"); new File("build/test/log").mkdirs(); System.setProperty("hadoop.log.dir", "build/test/log"); Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(4, fileSys.getUri().toString(), 1); jobConf = mr.createJobConf(); jobConf.set("mapred.child.java.opts", "-Xmx512m"); jobConf.setMapSpeculativeExecution(false); jobConf.setReduceSpeculativeExecution(false); } jobConf.setNumMapTasks(numMapTasks); jobConf.setNumReduceTasks(numReduceTasks); if (logger != null) properties.put("log4j.logger", logger); Flow.setJobPollingInterval(properties, 500); // should speed up tests MultiMapReducePlanner.setJobConf(properties, jobConf); }
From source file:cascading.platform.hadoop.HadoopPlatform.java
License:Open Source License
@Override public synchronized void setUp() throws IOException { if (configuration != null) return;//from w w w .ja v a 2 s . c om if (!isUseCluster()) { LOG.info("not using cluster"); configuration = new JobConf(); // enforce the local file system in local mode configuration.set("fs.default.name", "file:///"); configuration.set("mapred.job.tracker", "local"); configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/build/tmp/cascading/staging"); String stagingDir = configuration.get("mapreduce.jobtracker.staging.root.dir"); if (Util.isEmpty(stagingDir)) configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/build/tmp/cascading/staging"); fileSys = FileSystem.get(configuration); } else { LOG.info("using cluster"); if (Util.isEmpty(System.getProperty("hadoop.log.dir"))) System.setProperty("hadoop.log.dir", "cascading-hadoop/build/test/log"); if (Util.isEmpty(System.getProperty("hadoop.tmp.dir"))) System.setProperty("hadoop.tmp.dir", "cascading-hadoop/build/test/tmp"); new File(System.getProperty("hadoop.log.dir")).mkdirs(); // ignored JobConf conf = new JobConf(); if (!Util.isEmpty(System.getProperty("mapred.jar"))) { LOG.info("using a remote cluster with jar: {}", System.getProperty("mapred.jar")); configuration = conf; ((JobConf) configuration).setJar(System.getProperty("mapred.jar")); if (!Util.isEmpty(System.getProperty("fs.default.name"))) { LOG.info("using {}={}", "fs.default.name", System.getProperty("fs.default.name")); configuration.set("fs.default.name", System.getProperty("fs.default.name")); } if (!Util.isEmpty(System.getProperty("mapred.job.tracker"))) { LOG.info("using {}={}", "mapred.job.tracker", System.getProperty("mapred.job.tracker")); configuration.set("mapred.job.tracker", System.getProperty("mapred.job.tracker")); } configuration.set("mapreduce.user.classpath.first", "true"); // use test dependencies fileSys = FileSystem.get(configuration); } else { dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(4, fileSys.getUri().toString(), 1, null, null, conf); configuration = mr.createJobConf(); } // jobConf.set( "mapred.map.max.attempts", "1" ); // jobConf.set( "mapred.reduce.max.attempts", "1" ); configuration.set("mapred.child.java.opts", "-Xmx512m"); configuration.setInt("mapred.job.reuse.jvm.num.tasks", -1); configuration.setInt("jobclient.completion.poll.interval", 50); configuration.setInt("jobclient.progress.monitor.poll.interval", 50); ((JobConf) configuration).setMapSpeculativeExecution(false); ((JobConf) configuration).setReduceSpeculativeExecution(false); } ((JobConf) configuration).setNumMapTasks(numMappers); ((JobConf) configuration).setNumReduceTasks(numReducers); Map<Object, Object> globalProperties = getGlobalProperties(); if (logger != null) globalProperties.put("log4j.logger", logger); FlowProps.setJobPollingInterval(globalProperties, 10); // should speed up tests HadoopPlanner.copyProperties((JobConf) configuration, globalProperties); // copy any external properties HadoopPlanner.copyJobConf(properties, (JobConf) configuration); // put all properties on the jobconf }
From source file:cascading.platform.hadoop2.Hadoop2MR1Platform.java
License:Open Source License
@Override public synchronized void setUp() throws IOException { if (configuration != null) return;/*from www . j a v a2 s . c o m*/ if (!isUseCluster()) { LOG.info("not using cluster"); configuration = new JobConf(); // enforce settings to make local mode behave the same across distributions configuration.set("fs.defaultFS", "file:///"); configuration.set("mapreduce.framework.name", "local"); configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/" + "build/tmp/cascading/staging"); String stagingDir = configuration.get("mapreduce.jobtracker.staging.root.dir"); if (Util.isEmpty(stagingDir)) configuration.set("mapreduce.jobtracker.staging.root.dir", System.getProperty("user.dir") + "/build/tmp/cascading/staging"); fileSys = FileSystem.get(configuration); } else { LOG.info("using cluster"); if (Util.isEmpty(System.getProperty("hadoop.log.dir"))) System.setProperty("hadoop.log.dir", "build/test/log"); if (Util.isEmpty(System.getProperty("hadoop.tmp.dir"))) System.setProperty("hadoop.tmp.dir", "build/test/tmp"); new File(System.getProperty("hadoop.log.dir")).mkdirs(); // ignored JobConf conf = new JobConf(); if (!Util.isEmpty(System.getProperty("mapred.jar"))) { LOG.info("using a remote cluster with jar: {}", System.getProperty("mapred.jar")); configuration = conf; ((JobConf) configuration).setJar(System.getProperty("mapred.jar")); if (!Util.isEmpty(System.getProperty("fs.default.name"))) { LOG.info("using {}={}", "fs.default.name", System.getProperty("fs.default.name")); configuration.set("fs.default.name", System.getProperty("fs.default.name")); } if (!Util.isEmpty(System.getProperty("mapred.job.tracker"))) { LOG.info("using {}={}", "mapred.job.tracker", System.getProperty("mapred.job.tracker")); configuration.set("mapred.job.tracker", System.getProperty("mapred.job.tracker")); } if (!Util.isEmpty(System.getProperty("fs.defaultFS"))) { LOG.info("using {}={}", "fs.defaultFS", System.getProperty("fs.defaultFS")); configuration.set("fs.defaultFS", System.getProperty("fs.defaultFS")); } if (!Util.isEmpty(System.getProperty("yarn.resourcemanager.address"))) { LOG.info("using {}={}", "yarn.resourcemanager.address", System.getProperty("yarn.resourcemanager.address")); configuration.set("yarn.resourcemanager.address", System.getProperty("yarn.resourcemanager.address")); } if (!Util.isEmpty(System.getProperty("mapreduce.jobhistory.address"))) { LOG.info("using {}={}", "mapreduce.jobhistory.address", System.getProperty("mapreduce.jobhistory.address")); configuration.set("mapreduce.jobhistory.address", System.getProperty("mapreduce.jobhistory.address")); } configuration.set("mapreduce.user.classpath.first", "true"); // use test dependencies configuration.set("mapreduce.framework.name", "yarn"); fileSys = FileSystem.get(configuration); } else { conf.setBoolean("yarn.is.minicluster", true); // conf.setInt( "yarn.nodemanager.delete.debug-delay-sec", -1 ); // conf.set( "yarn.scheduler.capacity.root.queues", "default" ); // conf.set( "yarn.scheduler.capacity.root.default.capacity", "100" ); // disable blacklisting hosts not to fail localhost during unit tests conf.setBoolean("yarn.app.mapreduce.am.job.node-blacklisting.enable", false); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); FileSystem.setDefaultUri(conf, fileSys.getUri()); mr = MiniMRClientClusterFactory.create(this.getClass(), 4, conf); configuration = mr.getConfig(); } configuration.set("mapred.child.java.opts", "-Xmx512m"); configuration.setInt("mapreduce.job.jvm.numtasks", -1); configuration.setInt("mapreduce.client.completion.pollinterval", 50); configuration.setInt("mapreduce.client.progressmonitor.pollinterval", 50); configuration.setBoolean("mapreduce.map.speculative", false); configuration.setBoolean("mapreduce.reduce.speculative", false); } configuration.setInt("mapreduce.job.maps", numMappers); configuration.setInt("mapreduce.job.reduces", numReducers); Map<Object, Object> globalProperties = getGlobalProperties(); if (logger != null) globalProperties.put("log4j.logger", logger); FlowProps.setJobPollingInterval(globalProperties, 10); // should speed up tests Hadoop2MR1Planner.copyProperties(configuration, globalProperties); // copy any external properties Hadoop2MR1Planner.copyConfiguration(properties, configuration); // put all properties on the jobconf }
From source file:com.alexholmes.hadooputils.test.MiniHadoop.java
License:Apache License
/** * Creates a {@link MiniMRCluster} and {@link MiniDFSCluster} all working within * the directory supplied in {@code tmpDir}. * * The DFS will be formatted regardless if there was one or not before in the * given location.//from w w w . ja v a 2 s.c o m * * @param config the Hadoop configuration * @param taskTrackers number of task trackers to start * @param dataNodes number of data nodes to start * @param tmpDir the temporary directory which the Hadoop cluster will use for storage * @throws IOException thrown if the base directory cannot be set. */ public MiniHadoop(final Configuration config, final int taskTrackers, final int dataNodes, final File tmpDir) throws IOException { if (taskTrackers < 1) { throw new IllegalArgumentException("Invalid taskTrackers value, must be greater than 0"); } if (dataNodes < 1) { throw new IllegalArgumentException("Invalid dataNodes value, must be greater than 0"); } config.set("hadoop.tmp.dir", tmpDir.getAbsolutePath()); if (tmpDir.exists()) { FileUtils.forceDelete(tmpDir); } FileUtils.forceMkdir(tmpDir); // used by MiniDFSCluster for DFS storage System.setProperty("test.build.data", new File(tmpDir, "data").getAbsolutePath()); // required by JobHistory.initLogDir System.setProperty("hadoop.log.dir", new File(tmpDir, "logs").getAbsolutePath()); JobConf jobConfig = new JobConf(config); dfsCluster = new MiniDFSCluster(jobConfig, dataNodes, true, null); fileSystem = dfsCluster.getFileSystem(); mrCluster = new MiniMRCluster(0, 0, taskTrackers, fileSystem.getUri().toString(), 1, null, null, null, jobConfig); }
From source file:com.ibm.jaql.MiniCluster.java
License:Apache License
protected void setUp() throws IOException { final int numNodes = 1; conf = new Configuration(); if (System.getProperty("os.name").startsWith("Windows")) { // There is a bug in hadoop 0.20.1 on windows // ... INFO mapred.JobClient: Task Id : attempt_..., Status : FAILED // java.io.FileNotFoundException: // File C:/tmp/hadoop-xxx/mapred/local/1_0/taskTracker/jobcache/job_xxx/attempt_xxx/0_2/work/tmp // does not exist. // at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:361) // at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:245) // at org.apache.hadoop.mapred.TaskRunner.setupWorkDir(TaskRunner.java:519) // at org.apache.hadoop.mapred.Child.main(Child.java:155) ///*w w w .ja v a2 s . co m*/ // The following is reported to work around the problem String tmp = conf.get("hadoop.tmp.dir", "c:/temp"); conf.set("mapred.child.tmp", tmp + "/mapred/child.tmp"); } dfsCluster = new MiniDFSCluster(conf, numNodes, true, null); mrCluster = new MiniMRCluster(numNodes, dfsCluster.getFileSystem().getUri().getAuthority(), 1); setupOverride(mrCluster.createJobConf(), conf); // this.conf = conf = new Configuration(); // FileSystem fs = FileSystem.get(conf); // // make the home directory if it does not exist // Path hd = fs.getWorkingDirectory(); // if (!fs.exists(hd)) fs.mkdirs(hd); // // // make a tmp directory if it does not exist // Path t = new Path(tempDir); // if (!fs.exists(t)) // { // fs.mkdirs(t); // } }
From source file:com.inmobi.conduit.distcp.tools.mapred.lib.TestDynamicInputFormat.java
License:Apache License
@BeforeClass public static void setup() throws Exception { cluster = new MiniDFSCluster(getConfigurationForCluster(), 1, true, null); for (int i = 0; i < N_FILES; ++i) createFile("/tmp/source/" + String.valueOf(i)); }
From source file:com.inmobi.conduit.distcp.tools.mapred.TestCopyCommitter.java
License:Apache License
@BeforeClass public static void create() throws IOException { config = MockJobTracker.getJobForClient().getConfiguration(); counterProvider = new CounterProvider(EMPTY_COUNTERS); config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, 0); cluster = new MiniDFSCluster(config, 1, true, null); }