List of usage examples for org.apache.hadoop.mapred MiniMRCluster MiniMRCluster
public MiniMRCluster(int numTaskTrackers, String namenode, int numDir) throws IOException
From source file:JaqlShell.java
License:Apache License
/** * @param numNodes// w ww . j a v a 2 s . c o m * @param nameNode * @param conf * @return * @throws Exception */ private static MiniMRCluster startMRCluster(int numNodes, String nameNode, Configuration conf) throws Exception { MiniMRCluster mrc = new MiniMRCluster(numNodes, nameNode, 1); setupOverride(mrc, conf); return mrc; }
From source file:cascading.ClusterTestCase.java
License:Open Source License
public void setUp() throws IOException { if (jobConf != null) return;/*from w ww.j a va 2 s . c om*/ if (!enableCluster) { jobConf = new JobConf(); } else { System.setProperty("test.build.data", "build"); new File("build/test/log").mkdirs(); System.setProperty("hadoop.log.dir", "build/test/log"); Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(4, fileSys.getUri().toString(), 1); jobConf = mr.createJobConf(); jobConf.set("mapred.child.java.opts", "-Xmx512m"); jobConf.setMapSpeculativeExecution(false); jobConf.setReduceSpeculativeExecution(false); } jobConf.setNumMapTasks(numMapTasks); jobConf.setNumReduceTasks(numReduceTasks); if (logger != null) properties.put("log4j.logger", logger); Flow.setJobPollingInterval(properties, 500); // should speed up tests MultiMapReducePlanner.setJobConf(properties, jobConf); }
From source file:com.google.mr4c.hadoop.mrv1.MRv1TestBinding.java
License:Open Source License
private void startMRCluster() throws IOException { FileSystem fs = HadoopTestUtils.getTestDFS(); m_mrCluster = new MiniMRCluster(1, // # of task trackers fs.getUri().toString(), // name node 1 // # of directories );/* ww w. j a v a 2 s .c o m*/ }
From source file:com.ibm.jaql.MiniCluster.java
License:Apache License
protected void setUp() throws IOException { final int numNodes = 1; conf = new Configuration(); if (System.getProperty("os.name").startsWith("Windows")) { // There is a bug in hadoop 0.20.1 on windows // ... INFO mapred.JobClient: Task Id : attempt_..., Status : FAILED // java.io.FileNotFoundException: // File C:/tmp/hadoop-xxx/mapred/local/1_0/taskTracker/jobcache/job_xxx/attempt_xxx/0_2/work/tmp // does not exist. // at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:361) // at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:245) // at org.apache.hadoop.mapred.TaskRunner.setupWorkDir(TaskRunner.java:519) // at org.apache.hadoop.mapred.Child.main(Child.java:155) ///* ww w . j a v a 2 s. c o m*/ // The following is reported to work around the problem String tmp = conf.get("hadoop.tmp.dir", "c:/temp"); conf.set("mapred.child.tmp", tmp + "/mapred/child.tmp"); } dfsCluster = new MiniDFSCluster(conf, numNodes, true, null); mrCluster = new MiniMRCluster(numNodes, dfsCluster.getFileSystem().getUri().getAuthority(), 1); setupOverride(mrCluster.createJobConf(), conf); // this.conf = conf = new Configuration(); // FileSystem fs = FileSystem.get(conf); // // make the home directory if it does not exist // Path hd = fs.getWorkingDirectory(); // if (!fs.exists(hd)) fs.mkdirs(hd); // // // make a tmp directory if it does not exist // Path t = new Path(tempDir); // if (!fs.exists(t)) // { // fs.mkdirs(t); // } }
From source file:com.inmobi.conduit.distcp.tools.TestDistCp.java
License:Apache License
@BeforeClass public static void setup() throws Exception { configuration = getConfigurationForCluster(); cluster = new MiniDFSCluster(configuration, 1, true, null); System.setProperty("org.apache.hadoop.mapred.TaskTracker", "target/tmp"); configuration.set("org.apache.hadoop.mapred.TaskTracker", "target/tmp"); System.setProperty("hadoop.log.dir", "target/tmp"); configuration.set("hadoop.log.dir", "target/tmp"); mrCluster = new MiniMRCluster(1, configuration.get("fs.default.name"), 1); Configuration mrConf = mrCluster.createJobConf(); final String mrJobTracker = mrConf.get("mapred.job.tracker"); configuration.set("mapred.job.tracker", mrJobTracker); final String mrJobTrackerAddress = mrConf.get("mapred.job.tracker.http.address"); configuration.set("mapred.job.tracker.http.address", mrJobTrackerAddress); }
From source file:com.shopzilla.hadoop.mapreduce.MiniMRClusterContext.java
License:Apache License
@PostConstruct public void start() { try {/* w ww. ja va 2 s. c o m*/ this.hdfsRoot = new Path(localRoot.getFile().getName()); System.setProperty("hadoop.log.dir", logDirectory.getFilename()); System.setProperty("javax.xml.parsers.SAXParserFactory", "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"); miniDFSCluster = new MiniDFSCluster(configuration, 2, true, null); miniMrCluster = new MiniMRCluster(2, miniDFSCluster.getFileSystem().getUri().toString(), 1); File confFile = new File("/tmp/hadoop-site.xml"); configuration.setInt("mapred.submit.replication", 1); configuration.set("dfs.datanode.address", "0.0.0.0:0"); configuration.set("dfs.datanode.http.address", "0.0.0.0:0"); configuration.writeXml(new FileOutputStream(confFile)); System.setProperty("cluster", configuration.get("mapred.job.tracker")); System.setProperty("namenode", configuration.get("fs.default.name")); System.setProperty("junit.hadoop.conf", confFile.getPath()); pigServer = new PigServer(ExecType.MAPREDUCE, ConfigurationUtil.toProperties(configuration)); /*hiveServer = createHiveServer(); new Thread(new Runnable() { @Override public void run() { hiveServer.serve(); } }).start(); hiveClient = createHiveClient();*/ buildDirectory = new File(miniDFSCluster.getDataDirectory()).getParentFile().getParentFile() .getParentFile().getParentFile(); projectDirectory = buildDirectory.getParentFile(); importHDFSDirectory(localRoot.getFile()); } catch (Exception ex) { throw new RuntimeException(ex); } }
From source file:com.shopzilla.hadoop.testing.mapreduce.JobTracker.java
License:Apache License
@PostConstruct public JobTracker start() { try {/*from ww w .ja v a2 s .c o m*/ miniMrCluster = new MiniMRCluster(numTaskTrackers, dfsNameNode, 1); return this; } catch (final IOException ex) { throw new RuntimeException(ex); } }
From source file:datafu.hourglass.test.TestBase.java
License:Apache License
@SuppressWarnings("deprecation") public void beforeClass() throws Exception { // make sure the log folder exists or it will fail new File("test-logs").mkdirs(); System.setProperty("hadoop.log.dir", "test-logs"); if (localFS) { _fileSystem = FileSystem.get(new JobConf()); _log.info("*** Using local file system: " + _fileSystem.getUri()); } else {//from w ww .ja v a2 s. co m _log.info("*** Starting Mini DFS Cluster"); _dfsCluster = new MiniDFSCluster(new JobConf(), dataNodes, true, null); _fileSystem = _dfsCluster.getFileSystem(); } if (localMR) { _log.info("*** Using local MR Cluster"); } else { _log.info("*** Starting Mini MR Cluster"); _mrCluster = new MiniMRCluster(taskTrackers, _fileSystem.getName(), 1); } }
From source file:edu.berkeley.chukwa_xtrace.TestXtrExtract.java
License:Apache License
public void testArchiving() throws Exception { System.out.println("starting archive test"); Configuration conf = new Configuration(); System.setProperty("hadoop.log.dir", System.getProperty("test.build.data", "/tmp")); MiniDFSCluster dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null); FileSystem fileSys = dfs.getFileSystem(); fileSys.delete(OUTPUT_DIR, true);//nuke output dir writeASinkFile(conf, fileSys, INPUT_DIR, 1000); FileStatus fstat = fileSys.getFileStatus(INPUT_DIR); assertTrue(fstat.getLen() > 10);// ww w.ja v a2s . c om System.out.println("filesystem is " + fileSys.getUri()); conf.set("fs.default.name", fileSys.getUri().toString()); conf.setInt("io.sort.mb", 1); conf.setInt("io.sort.factor", 5); conf.setInt("mapred.tasktracker.map.tasks.maximum", 2); conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 2); MiniMRCluster mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1); String[] archiveArgs = { INPUT_DIR.toString(), fileSys.getUri().toString() + OUTPUT_DIR.toString() }; JobConf jc = mr.createJobConf(new JobConf(conf)); assertEquals("true", jc.get("archive.groupByClusterName")); assertEquals(1, jc.getInt("io.sort.mb", 5)); int returnVal = ToolRunner.run(jc, new XtrExtract(), archiveArgs); assertEquals(0, returnVal); fstat = fileSys.getFileStatus(new Path("/chukwa/archives/foocluster/HadoopLogProcessor_2008_05_29.arc")); assertTrue(fstat.getLen() > 10); Thread.sleep(1000); System.out.println("done!"); }
From source file:io.aos.t4f.hadoop.mapred.WordCountTest.java
License:Apache License
@Before public void init() throws IOException { new File(TARGET_FOLDER).mkdirs(); LOCAL_FILE.createNewFile();//from w ww .j a v a 2 s . c o m FileUtils.deleteDirectory(new File(MINI_DFS_CLUSTER_FOLDER)); new File(DATA_FOLDER).mkdirs(); new File(LOG_FOLDER).mkdirs(); configuration = new Configuration(); dfsCluster = new MiniDFSCluster(configuration, 1, true, null); System.setProperty("hadoop.log.dir", LOG_FOLDER); mrCluster = new MiniMRCluster(1, getFileSystem().getUri().toString(), 1); }